Merge pull request #31 from surface-security/upstream-to-pr/rev-344c54a

Upstream revision 344c54a

.github/workflows/build-test.yml

@@ -2,6 +2,9 @@ name: 🔨 Build Test
 
 on:
   pull_request:
+    paths:
+      - '**.go'
+      - '**.mod'
   workflow_dispatch:
 
 jobs:

.github/workflows/codeql-analysis.yml

@@ -3,13 +3,15 @@ name: 🚨 CodeQL Analysis
 on:
   workflow_dispatch:
   pull_request:
+    paths:
+      - '**.go'
     branches:
       - dev
 
 jobs:
   analyze:
     name: Analyze
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     permissions:
       actions: read
       contents: read

.github/workflows/dockerhub-push.yml

@@ -9,15 +9,15 @@ on:
 
 jobs:
   docker:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-16-cores
     steps:
       - name: Checkout
         uses: actions/checkout@v3
 
       - name: Get Github tag
         id: meta
         run: |
-          echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/httpx/releases/latest" | jq -r .tag_name)"
+          curl --silent "https://api.github.com/repos/projectdiscovery/httpx/releases/latest" | jq -r .tag_name | xargs -I {} echo TAG={} >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
@@ -32,9 +32,9 @@ jobs:
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/arm
           push: true
-          tags: projectdiscovery/httpx:latest,projectdiscovery/httpx:${{ steps.meta.outputs.tag }}
+          tags: projectdiscovery/httpx:latest,projectdiscovery/httpx:${{ steps.meta.outputs.TAG }}

.github/workflows/functional-test.yml

@@ -2,6 +2,9 @@ name: 🧪 Functional Test
 
 on:
   pull_request:
+    paths:
+      - '**.go'
+      - '**.mod'
   workflow_dispatch:
 
 jobs:  

.github/workflows/lint-test.yml

@@ -2,6 +2,9 @@ name: 🙏🏻 Lint Test
 
 on:
   pull_request:
+    paths:
+      - '**.go'
+      - '**.mod'
   workflow_dispatch:
 
 jobs:
@@ -16,7 +19,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.3.1
+        uses: golangci/golangci-lint-action@v3.4.0
         with:
           version: latest
           args: --timeout 5m

Dockerfile

@@ -1,11 +1,15 @@
-FROM golang:1.19.4-alpine AS builder
-ARG BUILD_SOURCE_TAG=latest
+# Base
+FROM golang:1.20.2-alpine AS builder
+
 RUN apk add --no-cache git build-base gcc musl-dev
-RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@${BUILD_SOURCE_TAG}
+WORKDIR /app
+COPY . /app
+RUN go mod download
+RUN go build ./cmd/httpx
 
-FROM alpine:3.17.0
+FROM alpine:3.17.2
 RUN apk -U upgrade --no-cache \
     && apk add --no-cache bind-tools ca-certificates
-COPY --from=builder /go/bin/httpx /usr/local/bin/
+COPY --from=builder /app/httpx /usr/local/bin/
 
-ENTRYPOINT ["httpx"]
+ENTRYPOINT ["httpx"]

README.md

@@ -115,7 +115,7 @@ MATCHERS:
    -mlc, -match-line-count string     match response body with specified line count (-mlc 423,532)
    -mwc, -match-word-count string     match response body with specified word count (-mwc 43,55)
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
-   -ms, -match-string string          match response with specified string (-ms admin)
+   -ms, -match-string string          match response with specified string (case insensitive) (-ms admin)
    -mr, -match-regex string           match response with specified regex (-mr admin)
    -mcdn, -match-cdn string[]         match host with specified cdn provider (oracle, google, azure, cloudflare, cloudfront, fastly, incapsula, leaseweb, akamai, sucuri)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')

cmd/functional-test/testcases.txt

@@ -16,4 +16,5 @@ scanme.sh {{binary}} -silent -body 'a=b'
 scanme.sh {{binary}} -silent -exclude-cdn
 scanme.sh {{binary}} -silent -ports https:443
 scanme.sh {{binary}} -silent -ztls
-https://scanme.sh?a=1*1 {{binary}} -silent
+https://scanme.sh?a=1*1 {{binary}} -silent
+https://scanme.sh:443 {{binary}} -asn

cmd/httpx/httpx.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/httpx/runner"
+	errorutil "github.com/projectdiscovery/utils/errors"
 )
 
 func main() {
@@ -58,3 +59,9 @@ func main() {
 	httpxRunner.RunEnumeration()
 	httpxRunner.Close()
 }
+
+func init() {
+	if os.Getenv("DEBUG") != "" {
+		errorutil.ShowStackTrace = true
+	}
+}

cmd/integration-test/library.go

@@ -44,7 +44,8 @@ func (h *httpxLibrary) Execute() error {
 
 	httpxRunner.RunEnumeration()
 
-	expected := "https://scanme.sh:443"
+	// httpx removes default ports for simplicity Ref: https://pkg.go.dev/github.com/projectdiscovery/httpx/common/stringz#RemoveURLDefaultPort
+	expected := "https://scanme.sh"
 
 	if got != expected {
 		return errIncorrectResult(expected, got)

common/httpx/httpx.go

@@ -19,6 +19,7 @@ import (
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
 	pdhttputil "github.com/projectdiscovery/utils/http"
 	stringsutil "github.com/projectdiscovery/utils/strings"
+	urlutil "github.com/projectdiscovery/utils/url"
 	"golang.org/x/net/context"
 	"golang.org/x/net/http2"
 )
@@ -331,11 +332,15 @@ func (h *HTTPX) NewRequest(method, targetURL string) (req *retryablehttp.Request
 
 // NewRequest from url
 func (h *HTTPX) NewRequestWithContext(ctx context.Context, method, targetURL string) (req *retryablehttp.Request, err error) {
-	req, err = retryablehttp.NewRequestWithContext(ctx, method, targetURL, nil)
+	urlx, err := urlutil.ParseURL(targetURL, h.Options.Unsafe)
 	if err != nil {
-		return
+		return nil, err
 	}
 
+	req, err = retryablehttp.NewRequestFromURLWithContext(ctx, method, urlx, nil)
+	if err != nil {
+		return nil, err
+	}
 	// Skip if unsafe is used
 	if !h.Options.Unsafe {
 		// set default user agent

common/stringz/stringz.go

@@ -93,19 +93,18 @@ func RemoveURLDefaultPort(rawURL string) string {
 		return rawURL
 	}
 
-	if u.Scheme == urlutil.HTTP && u.Port == "80" || u.Scheme == urlutil.HTTPS && u.Port == "443" {
-		u.Port = ""
+	if u.Scheme == urlutil.HTTP && u.Port() == "80" || u.Scheme == urlutil.HTTPS && u.Port() == "443" {
+		u.TrimPort()
 	}
 	return u.String()
 }
 
 func GetInvalidURI(rawURL string) (bool, string) {
 	if _, err := url.Parse(rawURL); err != nil {
 		if u, err := urlutil.Parse(rawURL); err == nil {
-			return true, u.RequestURI
+			return true, u.GetRelativePath()
 		}
 	}
-
 	return false, ""
 }