Merge pull request #1934 from projectdiscovery/dev

httpx v1.6.9 release prep

README.md

@@ -85,6 +85,12 @@ This will display help for the tool. Here are all the switches it supports.
 
 
 ```console
+Usage:
+  ./httpx [flags]
+
+Flags:
+httpx is a fast and multi-purpose HTTP toolkit that allows running multiple probes using the retryablehttp library.
+
 Usage:
   ./httpx [flags]
 
@@ -124,7 +130,8 @@ HEADLESS:
    -ho, -headless-options string[]  start headless chrome with additional options
    -esb, -exclude-screenshot-bytes  enable excluding screenshot bytes from json output
    -ehb, -exclude-headless-body     enable excluding headless header from json output
-   -st, -screenshot-timeout int     set timeout for screenshot in seconds (default 10)
+   -st, -screenshot-timeout value   set timeout for screenshot in seconds (default 10s)
+   -sid, -screenshot-idle value     set idle time before taking screenshot in seconds (default 1s)
 
 MATCHERS:
    -mc, -match-code string            match response with specified status code (-mc 200,302)
@@ -134,24 +141,25 @@ MATCHERS:
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
    -ms, -match-string string[]        match response with specified string (-ms admin)
    -mr, -match-regex string[]         match response with specified regex (-mr admin)
-   -mcdn, -match-cdn string[]         match host with specified cdn provider (google, cloudfront, fastly)
+   -mcdn, -match-cdn string[]         match host with specified cdn provider (cloudfront, fastly, google)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
    -mdc, -match-condition string      match response with dsl expression condition
 
 EXTRACTOR:
    -er, -extract-regex string[]   display response content with matched regex
-   -ep, -extract-preset string[]  display response content matched by a pre-defined regex (ipv4,mail,url)
+   -ep, -extract-preset string[]  display response content matched by a pre-defined regex (url,ipv4,mail)
 
 FILTERS:
    -fc, -filter-code string            filter response with specified status code (-fc 403,401)
    -fep, -filter-error-page            filter response with ML based error page detection
+   -fd, -filter-duplicates             filter out near-duplicate responses (only first response is retained)
    -fl, -filter-length string          filter response with specified content length (-fl 23,33)
    -flc, -filter-line-count string     filter response body with specified line count (-flc 423,532)
    -fwc, -filter-word-count string     filter response body with specified word count (-fwc 423,532)
    -ffc, -filter-favicon string[]      filter response with specified favicon hash (-ffc 1494302000)
    -fs, -filter-string string[]        filter response with specified string (-fs admin)
    -fe, -filter-regex string[]         filter response with specified regex (-fe admin)
-   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (google, cloudfront, fastly)
+   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (cloudfront, fastly, google)
    -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
    -fdc, -filter-condition string      filter response with dsl expression condition
    -strip                              strips all tags in response. supported formats: html,xml (default html)
@@ -234,6 +242,7 @@ DEBUG:
    -v, -verbose              verbose mode
    -si, -stats-interval int  number of seconds to wait between showing a statistics update (default: 5)
    -nc, -no-color            disable colors in cli output
+   -tr, -trace               trace
 
 OPTIMIZATIONS:
    -nf, -no-fallback                  display both probed protocol (HTTPS and HTTP)
@@ -248,7 +257,9 @@ OPTIMIZATIONS:
 
 CLOUD:
    -auth                           configure projectdiscovery cloud (pdcp) api key (default true)
+   -ac, -auth-config string        configure projectdiscovery cloud (pdcp) api key credential file
    -pd, -dashboard                 upload / view output in projectdiscovery cloud (pdcp) UI dashboard
+   -tid, -team-id string           upload asset results to given team id (optional)
    -aid, -asset-id string          upload new assets to existing asset id (optional)
    -aname, -asset-name string      assets group name to set (optional)
    -pdu, -dashboard-upload string  upload httpx output file (jsonl) in projectdiscovery cloud (pdcp) UI dashboard

cmd/httpx/httpx.go

@@ -138,5 +138,8 @@ func setupOptionalAssetUpload(opts *runner.Options) *pdcp.UploadWriter {
 		// silently ignore
 		writer.SetAssetGroupName(opts.AssetName)
 	}
+	if opts.TeamID != "" {
+		writer.SetTeamID(opts.TeamID)
+	}
 	return writer
 }

common/httpx/httpx.go

@@ -76,7 +76,7 @@ func New(options *Options) (*HTTPX, error) {
 	var retryablehttpOptions = retryablehttp.DefaultOptionsSpraying
 	retryablehttpOptions.Timeout = httpx.Options.Timeout
 	retryablehttpOptions.RetryMax = httpx.Options.RetryMax
-
+	retryablehttpOptions.Trace = options.Trace
 	handleHSTS := func(req *http.Request) {
 		if req.Response.Header.Get("Strict-Transport-Security") == "" {
 			return

common/httpx/option.go

@@ -49,6 +49,7 @@ type Options struct {
 	NetworkPolicy             *networkpolicy.NetworkPolicy
 	CDNCheckClient            *cdncheck.Client
 	Protocol                  Proto
+	Trace                     bool
 }
 
 // DefaultOptions contains the default options

common/errorpageclassifier/dataset.txt → common/pagetypeclassifier/dataset.txt

@@ -108,7 +108,7 @@ Now viewing: Image Gallery. Enjoy a visual tour of our activities.||nonerror
 You're on our FAQ page. Get answers to common questions.||nonerror
 Welcome to the Blog section. Engage with our thoughts and insights.||nonerror
 This is the Discussion Forum. Join in, ask questions, or help others.||nonerror
-You're on the Login page. Enter your credentials to access your account.||nonerror
+You're on the Login page. Enter your credentials to access your account.||login
 Welcome to the Sign-Up page. Join our community today.||nonerror
 This is your User Dashboard. Manage your account and settings here.||nonerror
 You've reached the Checkout page. Review your order and proceed to payment.||nonerror
@@ -198,4 +198,183 @@ You've successfully added the item to your cart!||nonerror
 Success! Your password has been updated||nonerror
 Welcome back! You have successfully logged in||nonerror
 Great job! Your profile has been updated||nonerror
-Your message was sent successfully. We'll get back to you shortly||nonerror
+Your message was sent successfully. We'll get back to you shortly||nonerror
+Welcome to the Login page. Please sign in to continue.||login
+Please enter your username and password on the login page.||login
+You have reached the login page. Access your account by logging in.||login
+Login required. Please authenticate to access this page.||login
+Welcome back! Please log in to your account.||login
+Sign in to your account on this login page.||login
+Secure Login: Enter your credentials to proceed.||login
+This is the login page. Please enter your email and password.||login
+Access denied. Please log in to continue.||login
+You're on the login page. Forgot your password? Click here to reset.||login
+User Login: Please provide your username and password.||login
+Login to your account to access exclusive features.||login
+Authentication required. Please log in.||login
+Welcome back! Sign in to access your dashboard.||login
+Please log in to proceed to the checkout page.||login
+Member login: Enter your credentials below.||login
+Staff login portal. Please enter your login details.||login
+Customer login: Sign in to view your orders.||login
+Partner login: Please authenticate to access partner resources.||login
+Administrator login page. Enter your admin credentials.||login
+Please log in to access your profile settings.||login
+Login successful. Redirecting to your account dashboard.||login
+Incorrect password. Please try again.||login
+Session expired. Please log in again.||login
+Welcome to the secure login page. Your privacy is important to us.||login
+Access restricted. Please log in to view this content.||login
+Please log in to access the members-only area.||login
+Sign in with your social media account on the login page.||login
+New user? Register here or log in if you already have an account.||login
+Log in to participate in the forum discussions.||login
+Access your account by logging in here.||login
+Please log in to access your personalized dashboard.||login
+Enter your login details to continue.||login
+Login Page: Securely enter your credentials.||login
+Welcome to the user login portal.||login
+Sign in to manage your account settings.||login
+This is the login screen. Please authenticate.||login
+Returning user? Please log in.||login
+Please log in to view your messages.||login
+Log in to access premium content.||login
+Authentication page: Enter your username and password.||login
+Please enter your login information to proceed.||login
+User authentication required. Please log in.||login
+Log in now to unlock exclusive features.||login
+Sign in to check your account balance.||login
+Welcome back! Please enter your login credentials.||login
+Member login area: Access restricted content by logging in.||login
+Please sign in to continue to your profile.||login
+Staff members, please log in to access internal resources.||login
+Enter your email and password to log in.||login
+Login required to view this page. Please sign in.||login
+Access your profile by logging into your account.||login
+Please provide your login credentials to access the system.||login
+Log in to track your order status.||login
+Welcome to the employee login page.||login
+Secure area: Please log in to continue.||login
+Please log in to update your preferences.||login
+Sign in to access your learning materials.||login
+Please authenticate to proceed to the next step.||login
+Login Page: Your session has expired, please log in again.||login
+Welcome back! Enter your credentials to sign in.||login
+Client login: Access your project details here.||login
+Agent login portal: Please sign in with your ID.||login
+Enter your user ID and password to log in.||login
+Log in to view your subscription details.||login
+Login Page: Forgot your password? Click here to reset it.||login
+Access restricted to authorized users only. Please log in.||login
+Vendor login: Manage your listings by logging in.||login
+Please log in to access your saved items.||login
+Log in to participate in our online courses.||login
+Sign in to view your appointment schedule.||login
+Welcome to the admin login page.||login
+Please enter your credentials to log in securely.||login
+Log in to view your recent activities.||login
+Authentication needed. Please sign in to proceed.||login
+Member login: Keep me signed in checkbox available.||login
+Log in with your email or username.||login
+Access your account dashboard by logging in.||login
+Sign in to post comments on articles.||login
+Please log in to access your billing information.||login
+Log in to access your personalized recommendations.||login
+Please sign in to view your shopping cart.||login
+Enter your credentials to log in and start shopping.||login
+Welcome to the customer login page. Sign in to continue.||login
+Authentication required. Please log in with your secure ID.||login
+Log in to access exclusive member discounts.||login
+Please log in to view and manage your wishlist.||login
+Sign in to access your event tickets and details.||login
+Faculty login: Please enter your staff ID and password.||login
+Log in to access your investment portfolio.||login
+Access your medical records by logging in securely.||login
+Please sign in to continue to the payment gateway.||login
+Login required to access your order history.||login
+Welcome back! Log in to resume your session.||login
+Please log in to submit your application.||login
+Enter your username and password to log in to the portal.||login
+Student login: Access your course materials by signing in.||login
+Log in to customize your news feed preferences.||login
+Please authenticate to access your secure messages.||login
+Sign in to sync your data across devices.||login
+Log in to join the live webinar.||login
+Please log in to access your reservation details.||login
+Welcome to the supplier login page.||login
+Log in to access your support tickets.||login
+Enter your credentials to log in and view analytics.||login
+Please sign in to access developer resources.||login
+Login required to view confidential documents.||login
+Log in to participate in the survey.||login
+Please authenticate to access the admin dashboard.||login
+Sign in to view your loyalty points balance.||login
+Log in to manage your email subscriptions.||login
+Please log in to proceed with the enrollment process.||login
+Access your download history by logging in.||login
+Welcome back! Please log in to renew your membership.||login
+Enter your employee ID to log in to the time tracking system.||login
+Log in to update your security settings.||login
+Please sign in to access your saved searches.||login
+Authentication required for accessing project files.||login
+Log in to collaborate with your team members.||login
+Please enter your PIN and password to log in.||login
+Sign in to access your fitness progress dashboard.||login
+Log in to check your test results.||login
+Please log in to schedule your appointments.||login
+Welcome to the volunteer login page.||login
+Log in to view your donation history.||login
+Please authenticate to access the control panel.||login
+Sign in to review and accept your job offer.||login
+Log in to access premium tutorials and guides.||login
+Please log in to manage your API keys.||login
+Please log in with your email address and password.||login
+Enter your username and password to access your account.||login
+Sign in to your account using your email and password.||login
+Welcome back! Please enter your login credentials.||login
+Email address: [input field] Password: [input field]||login
+Login to your account. Don't have one? Sign up now.||login
+Username: [input field] Password: [input field] Remember me?||login
+Forgot your password? Click here to reset it.||login
+Please enter your email and password to continue.||login
+Secure login portal. Enter credentials below.||login
+Access your account by logging in below.||login
+Remember me on this device.||login
+Login required. Please sign in to proceed.||login
+Forgot password? Reset it here.||login
+Sign in with your email address and password.||login
+Welcome! Please log in to your account.||login
+User login: Enter your email and password.||login
+Email: [input field] Password: [input field]||login
+Please authenticate by entering your login details.||login
+Sign in to your account or register for a new one.||login
+Login page: Access restricted to authorized users only.||login
+Need help logging in? Click here.||login
+Enter your credentials to log in.||login
+Keep me signed in.||login
+Please sign in to access exclusive content.||login
+Welcome back! Sign in to your dashboard.||login
+Forgot your username or password? Retrieve them here.||login
+Log in using your email or username.||login
+Authentication required. Please log in.||login
+Password recovery: Reset your password now.||login
+Log in to manage your account settings.||login
+Sign in to continue to checkout.||login
+Enter your login information below.||login
+Sign in to access your personalized dashboard.||login
+Welcome to the member login page.||login
+Already have an account? Log in here.||login
+Enter email and password to sign in.||login
+Sign in to your profile.||login
+Member login: Access your account here.||login
+Please log in to continue.||login
+Enter your password to log in.||login
+Sign in to view your messages.||login
+Login to your profile to see updates.||login
+Log in to your account to access features.||login
+Please provide your username and password.||login
+Log in to manage your subscriptions.||login
+Sign in using your credentials.||login
+Access denied. Please log in first.||login
+Authentication portal. Enter login details.||login
+Need an account? Sign up or log in if you already have one.||login

common/errorpageclassifier/errorpageclassifier.go → common/pagetypeclassifier/pagetypeclassifier.go

@@ -1,4 +1,4 @@
-package errorpageclassifier
+package pagetypeclassifier
 
 import (
 	_ "embed"
@@ -10,19 +10,19 @@ import (
 //go:embed clf.gob
 var classifierData []byte
 
-type ErrorPageClassifier struct {
+type PageTypeClassifier struct {
 	classifier *naive_bayes.NaiveBayesClassifier
 }
 
-func New() *ErrorPageClassifier {
+func New() *PageTypeClassifier {
 	classifier, err := naive_bayes.NewClassifierFromFileData(classifierData)
 	if err != nil {
 		panic(err)
 	}
-	return &ErrorPageClassifier{classifier: classifier}
+	return &PageTypeClassifier{classifier: classifier}
 }
 
-func (n *ErrorPageClassifier) Classify(html string) string {
+func (n *PageTypeClassifier) Classify(html string) string {
 	text := htmlToText(html)
 	if text == "" {
 		return "other"

common/errorpageclassifier/errorpageclassifier_test.go → common/pagetypeclassifier/pagetypeclassifier_test.go

@@ -1,13 +1,14 @@
-package errorpageclassifier
+package pagetypeclassifier
 
 import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestErrorPageClassifier(t *testing.T) {
-	t.Run("test creation of new ErrorPageClassifier", func(t *testing.T) {
+func TestPageTypeClassifier(t *testing.T) {
+
+	t.Run("test creation of new PageTypeClassifier", func(t *testing.T) {
 		epc := New()
 		assert.NotNil(t, epc)
 	})