Adding Screenshot support

.github/workflows/build-test.yml

@@ -31,6 +31,10 @@ jobs:
         run: go test ./...
         working-directory: .
 
+      - name: Running example
+        run: go run .
+        working-directory: examples/
+
       - name: Integration Tests Linux, macOS
         if: runner.os == 'Linux' || runner.os == 'macOS'
         env:

Dockerfile

@@ -9,7 +9,7 @@ RUN go build ./cmd/httpx
 
 FROM alpine:3.17.3
 RUN apk -U upgrade --no-cache \
-    && apk add --no-cache bind-tools ca-certificates
+    && apk add --no-cache bind-tools ca-certificates chromium
 COPY --from=builder /app/httpx /usr/local/bin/
 
 ENTRYPOINT ["httpx"]

README.md

@@ -109,15 +109,19 @@ PROBES:
    -cdn                  display cdn in use
    -probe                display probe status
 
+HEADLESS:
+   -ss, -screenshot  enable saving screenshot of the page using headless browser
+   -system-chrome    enable using local installed chrome for screenshot
+
 MATCHERS:
    -mc, -match-code string            match response with specified status code (-mc 200,302)
    -ml, -match-length string          match response with specified content length (-ml 100,102)
    -mlc, -match-line-count string     match response body with specified line count (-mlc 423,532)
    -mwc, -match-word-count string     match response body with specified word count (-mwc 43,55)
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
-   -ms, -match-string string          match response with specified string (case insensitive) (-ms admin)
+   -ms, -match-string string          match response with specified string (-ms admin)
    -mr, -match-regex string           match response with specified regex (-mr admin)
-   -mcdn, -match-cdn string[]         match host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
+   -mcdn, -match-cdn string[]         match host with specified cdn provider (incapsula, oracle, google, azure, cloudflare, cloudfront, fastly, akamai, sucuri, leaseweb)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
    -mdc, -match-condition string      match response with dsl expression condition
 
@@ -133,7 +137,7 @@ FILTERS:
    -ffc, -filter-favicon string[]      filter response with specified favicon hash (-mfc 1494302000)
    -fs, -filter-string string          filter response with specified string (-fs admin)
    -fe, -filter-regex string           filter response with specified regex (-fe admin)
-   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
+   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (incapsula, oracle, google, azure, cloudflare, cloudfront, fastly, akamai, sucuri, leaseweb)
    -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
    -fdc, -filter-condition string      filter response with dsl expression condition
 
@@ -154,6 +158,10 @@ MISCELLANEOUS:
    -vhost                     probe and display server supporting VHOST
    -ldv, -list-dsl-variables  list json output field keys name that support dsl matcher/filter
 
+UPDATE:
+   -up, -update                 update httpx to latest version
+   -duc, -disable-update-check  disable automatic httpx update check
+
 OUTPUT:
    -o, -output string                  file to write output results
    -sr, -store-response                store http response to output directory
@@ -184,7 +192,7 @@ CONFIGURATIONS:
    -body string                  post body to include in http request
    -s, -stream                   stream mode - start elaborating input targets without sorting
    -sd, -skip-dedupe             disable dedupe input items (only used with stream mode)
-   -ldp, -leave-default-ports    leave default http/https ports in host header (eg. http://host:80 - https//host:443
+   -ldp, -leave-default-ports    leave default http/https ports in host header (eg. http://host:80 - https://host:443
    -ztls                         use ztls library with autofallback to standard one for tls13
    -no-decode                    avoid decoding body
 
@@ -472,55 +480,75 @@ https://docs.hackerone.com
 https://support.hackerone.com
 ```
 
-### Using `httpx` as a library
-`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it:
+### Screenshot
 
-```go
-package main
+Latest addition to the project, the addition of the `-screenshot` option in httpx, a powerful new feature that allows users to take screenshots of target URLs, pages, or endpoints along with the rendered DOM. This functionality enables the **visual content discovery process**, providing a comprehensive view of the target's visual appearance.
 
-import (
-	"log"
+Rendered DOM body is also included in json line output when `-screenshot` option is used with `-json` option.
 
-	"github.com/projectdiscovery/goflags"
-	"github.com/projectdiscovery/gologger"
-	"github.com/projectdiscovery/gologger/levels"
-	"github.com/projectdiscovery/httpx/runner"
-)
+#### 🚩 Usage
 
-func main() {
-	gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional)
+To use the screenshot feature, simply add the `-screenshot` flag to your httpx command:
 
-	options := runner.Options{
-		Methods: "GET",
-		InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"},
-		//InputFile: "./targetDomains.txt", // path to file containing the target domains list 
-	}
+```console
+httpx -screenshot -u https://example.com
+```
 
-	if err := options.ValidateOptions(); err != nil {
-		log.Fatal(err)
-	}
+🎯 Domain, Subdomain, and Path Support
+The `-screenshot` option is versatile and can be used to capture screenshots for domains, subdomains, and even specific paths when used in conjunction with the `-path` option:
 
-	httpxRunner, err := runner.New(&options)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer httpxRunner.Close()
+```console
+httpx -screenshot -u example.com
+httpx -screenshot -u https://example.com/login
+httpx -screenshot -path fuzz_path.txt -u https://example.com
+```
 
-	httpxRunner.RunEnumeration()
-}
+Using with other tools:
+
+```console
+subfinder -d example.com | httpx -screenshot
 ```
 
+#### 🌐 System Chrome
+
+By default, httpx will use the go-rod library to install and manage Chrome for taking screenshots. However, if you prefer to use your locally installed system Chrome, add the `-system-chrome` flag:
+
+```console
+httpx -screenshot -system-chrome -u https://example.com
+```
+
+#### 📁 Output Directory
+
+Screenshots are stored in the output/screenshot directory by default. To specify a custom output directory, use the `-srd` option:
+
+```console
+httpx -screenshot -srd /path/to/custom/directory -u https://example.com
+```
+
+#### ⏳ Performance Considerations
+
+Please note that since screenshots are captured using a headless browser, httpx runs will be slower when using the `-screenshot` option.
+
+### Using `httpx` as a library
+`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to be closed at the end of the program) and the `RunEnumeration` method should be called. A minimal example of how to do it is in the [examples](examples/) folder
 
 # Notes
 
-- As default, `httpx` checks for **HTTPS** probe and fall-back to **HTTP** only if **HTTPS** is not reachable.
-- The `-no-fallback` flag can be used to display both **HTTP** and **HTTPS** results 
+- As default, `httpx` probe with **HTTPS** scheme and fall-back to **HTTP** only if **HTTPS** is not reachable.
+- The `-no-fallback` flag can be used to probe and display both **HTTP** and **HTTPS** result.
 - Custom scheme for ports can be defined, for example `-ports http:443,http:80,https:8443`
-- The following flags should be used for specific use cases instead of running them as default with other probes:
-  * `-favicon`,`-vhost`, `-http2`, `-pipeline`, `-ports`, `-csp-probe`, `-tls-probe`, `-path`
-- When using the `-json` flag, all the default probe results are included in the JSON output.
 - Custom resolver supports multiple protocol (**doh|tcp|udp**) in form of `protocol:resolver:port` (e.g. `udp:127.0.0.1:53`)
-- Invalid custom resolvers/files are ignored.
+- The following flags should be used for specific use cases instead of running them as default with other probes:
+   - `-ports`
+   - `-path`
+   - `-vhost`
+   - `-screenshot`
+   - `-csp-probe`
+   - `-tls-probe`
+   - `-favicon`
+   - `-http2`
+   - `-pipeline`
+
 
 # Acknowledgement
 

common/fileutil/fileutil.go

@@ -80,3 +80,10 @@ func LoadCidrsFromSliceOrFileWithMaxRecursion(option string, splitchar string, m
 
 	return
 }
+
+func AbsPathOrDefault(p string) string {
+	if absPath, err := filepath.Abs(p); err == nil {
+		return absPath
+	}
+	return p
+}

examples/example.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"log"
+
+	"github.com/projectdiscovery/goflags"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/gologger/levels"
+	"github.com/projectdiscovery/httpx/runner"
+)
+
+func main() {
+	gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) // increase the verbosity (optional)
+
+	options := runner.Options{
+		Methods:         "GET",
+		InputTargetHost: goflags.StringSlice{"scanme.sh", "projectdiscovery.io"},
+		//InputFile: "./targetDomains.txt", // path to file containing the target domains list
+	}
+
+	if err := options.ValidateOptions(); err != nil {
+		log.Fatal(err)
+	}
+
+	httpxRunner, err := runner.New(&options)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer httpxRunner.Close()
+
+	httpxRunner.RunEnumeration()
+}

go.mod

@@ -4,7 +4,6 @@ go 1.19
 
 require (
 	github.com/akrylysov/pogreb v0.10.1 // indirect
-	github.com/bluele/gcache v0.0.2
 	github.com/corpix/uarand v0.2.0
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
@@ -37,8 +36,10 @@ require (
 require github.com/spaolacci/murmur3 v1.1.0
 
 require (
+	github.com/Mzack9999/gcache v0.0.0-20230410081825-519e28eab057
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/bxcodec/faker/v4 v4.0.0-beta.3
+	github.com/go-rod/rod v0.112.8
 	github.com/hdm/jarm-go v0.0.7
 	github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6
 	github.com/mitchellh/mapstructure v1.5.0
@@ -55,6 +56,7 @@ require (
 
 require (
 	aead.dev/minisign v0.2.0 // indirect
+	cloud.google.com/go/compute/metadata v0.2.0 // indirect
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect
@@ -86,6 +88,7 @@ require (
 	github.com/fatih/color v1.14.1 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
 	github.com/fullstorydev/grpcurl v1.8.1 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/mock v1.5.0 // indirect
@@ -107,6 +110,7 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kataras/jwt v0.1.8 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
@@ -121,6 +125,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20230328171319-f24b18d05b64 // indirect
 	github.com/projectdiscovery/freeport v0.0.4 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.4 // indirect
@@ -133,6 +138,8 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/sashabaranov/go-openai v1.8.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.3 // indirect
+	github.com/shoenig/go-m1cpu v0.1.4 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/spf13/cobra v1.1.3 // indirect
@@ -146,6 +153,8 @@ require (
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/rtred v0.1.2 // indirect
 	github.com/tidwall/tinyqueue v0.1.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.11 // indirect
+	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 // indirect
@@ -154,8 +163,12 @@ require (
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
+	github.com/ysmood/goob v0.4.0 // indirect
+	github.com/ysmood/gson v0.7.3 // indirect
+	github.com/ysmood/leakless v0.8.0 // indirect
 	github.com/yuin/goldmark v1.5.4 // indirect
 	github.com/yuin/goldmark-emoji v1.0.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	github.com/zmap/rc2 v0.0.0-20190804163417-abaa70531248 // indirect
 	github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.0-alpha.0 // indirect