Make cdn and tech-detect as default in json output (#1614)

* Make cdn and tech-detect as default in json output * Reflect changes only in jsonl output
projectdiscovery · Mar 6, 2024 · dd8b546 · dd8b546
1 parent daf7f01
commit dd8b546
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 28 deletions.
diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go
@@ -74,7 +74,7 @@ func (h *httpxLibraryWithStream) Execute() error {
 		RateLimit:  150,
 		Retries:    2,
 		Timeout:    10,
-		TechDetect: true,
+		TechDetect: "true",
 		Stream:     true,
 		SkipDedupe: true,
 		OnResult: func(r runner.Result) {

diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go
@@ -187,7 +187,7 @@ func New(options *Options) (*HTTPX, error) {
 
 	httpx.htmlPolicy = bluemonday.NewPolicy()
 	httpx.CustomHeaders = httpx.Options.CustomHeaders
-	if options.CdnCheck || options.ExcludeCdn {
+	if options.CdnCheck != "false" || options.ExcludeCdn {
 		httpx.cdn = cdncheck.New()
 	}
 

diff --git a/common/httpx/option.go b/common/httpx/option.go
@@ -15,7 +15,7 @@ type Options struct {
 	HTTPProxy        string
 	SocksProxy       string
 	Threads          int
-	CdnCheck         bool
+	CdnCheck         string
 	ExcludeCdn       bool
 	// Timeout is the maximum time to wait for the request
 	Timeout time.Duration
@@ -55,7 +55,7 @@ var DefaultOptions = Options{
 	RetryMax:     5,
 	MaxRedirects: 10,
 	Unsafe:       false,
-	CdnCheck:     true,
+	CdnCheck:     "true",
 	ExcludeCdn:   false,
 	// VHOSTs options
 	VHostIgnoreStatusCode:    false,

diff --git a/runner/options.go b/runner/options.go
@@ -69,12 +69,12 @@ type ScanOptions struct {
 	HTTP2Probe                bool
 	OutputIP                  bool
 	OutputCName               bool
-	OutputCDN                 bool
+	OutputCDN                 string
 	OutputResponseTime        bool
 	PreferHTTPS               bool
 	NoFallback                bool
 	NoFallbackScheme          bool
-	TechDetect                bool
+	TechDetect                string
 	StoreChain                bool
 	StoreVisionReconClusters  bool
 	MaxResponseBodySizeToSave int
@@ -225,11 +225,11 @@ type Options struct {
 	DebugResponse             bool
 	Pipeline                  bool
 	HTTP2Probe                bool
-	OutputCDN                 bool
+	OutputCDN                 string
 	OutputResponseTime        bool
 	NoFallback                bool
 	NoFallbackScheme          bool
-	TechDetect                bool
+	TechDetect                string
 	TLSGrab                   bool
 	protocol                  string
 	ShowStatistics            bool
@@ -327,13 +327,13 @@ func ParseOptions() *Options {
 		flagSet.BoolVar(&options.ExtractTitle, "title", false, "display page title"),
 		flagSet.DynamicVarP(&options.ResponseBodyPreviewSize, "body-preview", "bp", 100, "display first N characters of response body"),
 		flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "display server name"),
-		flagSet.BoolVarP(&options.TechDetect, "tech-detect", "td", false, "display technology in use based on wappalyzer dataset"),
+		flagSet.DynamicVarP(&options.TechDetect, "tech-detect", "td", "true", "display technology in use based on wappalyzer dataset"),
 		flagSet.BoolVar(&options.OutputMethod, "method", false, "display http request method"),
 		flagSet.BoolVar(&options.OutputWebSocket, "websocket", false, "display server using websocket"),
 		flagSet.BoolVar(&options.OutputIP, "ip", false, "display host ip"),
 		flagSet.BoolVar(&options.OutputCName, "cname", false, "display host cname"),
 		flagSet.BoolVar(&options.Asn, "asn", false, "display host asn information"),
-		flagSet.BoolVar(&options.OutputCDN, "cdn", false, "display cdn/waf in use"),
+		flagSet.DynamicVar(&options.OutputCDN, "cdn", "true", "display cdn/waf in use"),
 		flagSet.BoolVar(&options.Probe, "probe", false, "display probe status"),
 	)
 
@@ -665,7 +665,7 @@ func (options *Options) ValidateOptions() error {
 		}
 	}
 	if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 {
-		options.OutputCDN = true
+		options.OutputCDN = "true"
 	}
 
 	return nil

diff --git a/runner/runner.go b/runner/runner.go
@@ -106,7 +106,7 @@ func New(options *Options) (*Runner, error) {
 		options: options,
 	}
 	var err error
-	if options.TechDetect {
+	if options.TechDetect != "false" {
 		runner.wappalyzer, err = wappalyzer.New()
 	}
 	if err != nil {
@@ -1756,7 +1756,7 @@ retry:
 	}
 
 	isCDN, cdnName, err := hp.CdnCheck(ip)
-	if scanopts.OutputCDN && isCDN && err == nil {
+	if scanopts.OutputCDN == "true" && isCDN && err == nil {
 		builder.WriteString(fmt.Sprintf(" [%s]", cdnName))
 	}
 
@@ -1765,24 +1765,24 @@ retry:
 	}
 
 	var technologies []string
-	if scanopts.TechDetect {
+	if scanopts.TechDetect != "false" {
 		matches := r.wappalyzer.Fingerprint(resp.Headers, resp.Data)
 		for match := range matches {
 			technologies = append(technologies, match)
 		}
+	}
 
-		if len(technologies) > 0 {
-			sort.Strings(technologies)
-			technologies := strings.Join(technologies, ",")
+	if scanopts.TechDetect == "true" && len(technologies) > 0 {
+		sort.Strings(technologies)
+		technologies := strings.Join(technologies, ",")
 
-			builder.WriteString(" [")
-			if !scanopts.OutputWithNoColor {
-				builder.WriteString(aurora.Magenta(technologies).String())
-			} else {
-				builder.WriteString(technologies)
-			}
-			builder.WriteRune(']')
+		builder.WriteString(" [")
+		if !scanopts.OutputWithNoColor {
+			builder.WriteString(aurora.Magenta(technologies).String())
+		} else {
+			builder.WriteString(technologies)
 		}
+		builder.WriteRune(']')
 	}
 
 	var extractRegex []string
@@ -1831,10 +1831,6 @@ retry:
 		}
 	}
 
-	// adding default hashing for json output format
-	if r.options.JSONOutput && len(scanopts.Hashes) == 0 {
-		scanopts.Hashes = "md5,mmh3,sha256,simhash"
-	}
 	hashesMap := make(map[string]interface{})
 	if scanopts.Hashes != "" {
 		hs := strings.Split(scanopts.Hashes, ",")