Merge pull request #22 from surface-security/dev-126

Update to v1.2.6

.github/workflows/lint-test.yml

@@ -0,0 +1,23 @@
+name: 🙏🏻 Lint Test
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Run golangci-lint
+        uses: golangci/[email protected]
+        with:
+          version: latest
+          args: --timeout 5m
+          working-directory: .

.gitignore

@@ -1,5 +1,10 @@
-.vscode
+.idea/
+.vscode/
 cmd/httpx/httpx
 /test/output
 integration_tests/httpx
 integration_tests/integration-test
+cmd/functional-test/httpx_dev
+cmd/functional-test/functional-test
+cmd/functional-test/httpx
+cmd/functional-test/*.cfg

.goreleaser.yml

@@ -20,6 +20,8 @@ builds:
       goarch: '386'
     - goos: windows
       goarch: 'arm'
+    - goos: windows
+      goarch: 'arm64'
 
   binary: '{{ .ProjectName }}'
   main: cmd/httpx/httpx.go
@@ -31,3 +33,14 @@ archives:
 
 checksum:
   algorithm: sha256
+
+announce:
+  slack:
+    enabled: true
+    channel: '#release'
+    username: GoReleaser
+    message_template: 'New Release: {{ .ProjectName }} {{ .Tag }} is published! Check it out at {{ .ReleaseURL }}'
+
+  discord:
+    enabled: true
+    message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}'

Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:1.17.0-alpine AS builder
+FROM golang:1.19.4-alpine AS builder
 WORKDIR /go/src/httpx
 ADD go.mod .
 ADD go.sum .
 RUN go mod download
 ADD . .
-RUN go build -v -ldflags="-s -w" -o "/httpx" cmd/httpx/httpx.go
+RUN CGO_ENABLED=0 go build -v -ldflags="-s -w" -o "/httpx" cmd/httpx/httpx.go
 
 
 FROM alpine:3.12

cmd/functional-test/main.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/pkg/errors"
+
+	"github.com/projectdiscovery/httpx/internal/testutils"
+)
+
+var (
+	debug   = os.Getenv("DEBUG") == "true"
+	success = aurora.Green("[✓]").String()
+	failed  = aurora.Red("[✘]").String()
+	errored = false
+
+	mainHttpxBinary = flag.String("main", "", "Main Branch Httpx Binary")
+	devHttpxBinary  = flag.String("dev", "", "Dev Branch Httpx Binary")
+	testcases       = flag.String("testcases", "", "Test cases file for Httpx functional tests")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := runFunctionalTests(); err != nil {
+		log.Fatalf("Could not run functional tests: %s\n", err)
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func runFunctionalTests() error {
+	file, err := os.Open(*testcases)
+	if err != nil {
+		return errors.Wrap(err, "could not open test cases")
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		text := strings.TrimSpace(scanner.Text())
+		if text == "" {
+			continue
+		}
+		if err := runIndividualTestCase(text); err != nil {
+			errored = true
+			fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, text, err)
+		} else {
+			fmt.Printf("%s Test \"%s\" passed!\n", success, text)
+		}
+	}
+	return nil
+}
+
+func runIndividualTestCase(testcase string) error {
+	parts := strings.Fields(testcase)
+
+	var finalArgs []string
+	var target string
+	if len(parts) > 1 {
+		finalArgs = parts[2:]
+		target = parts[0]
+	}
+	mainOutput, err := testutils.RunHttpxBinaryAndGetResults(target, *mainHttpxBinary, debug, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run httpx main test")
+	}
+	devOutput, err := testutils.RunHttpxBinaryAndGetResults(target, *devHttpxBinary, debug, finalArgs)
+	if err != nil {
+		return errors.Wrap(err, "could not run httpx dev test")
+	}
+	if len(mainOutput) == len(devOutput) {
+		return nil
+	}
+	return fmt.Errorf("%s main is not equal to %s dev", mainOutput, devOutput)
+}

cmd/functional-test/run.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# reading os type from arguments
+CURRENT_OS=$1
+
+if [ "${CURRENT_OS}" == "windows-latest" ];then
+    extension=.exe
+fi
+
+echo "::group::Building functional-test binary"
+go build -o functional-test$extension
+echo "::endgroup::"
+
+echo "::group::Building httpx binary from current branch"
+go build -o httpx_dev$extension ../httpx
+echo "::endgroup::"
+
+echo "::group::Building latest release of httpx"
+go build -o httpx$extension -v github.com/projectdiscovery/httpx/cmd/httpx
+echo "::endgroup::"
+
+echo 'Starting httpx functional test'
+./functional-test$extension -main ./httpx$extension -dev ./httpx_dev$extension -testcases testcases.txt

cmd/functional-test/test-data/raw-request.txt

@@ -0,0 +1 @@
+GET /search?q=test HTTP/2

cmd/functional-test/test-data/request.txt

@@ -0,0 +1 @@
+https://scanme.sh

cmd/functional-test/testcases.txt

@@ -0,0 +1,19 @@
+scanme.sh {{binary}} -silent 
+scanme.sh {{binary}} -silent -l test-data/request.txt
+scanme.sh {{binary}} -silent -request test-data/raw-request.txt
+scanme.sh {{binary}} -silent -title
+scanme.sh {{binary}} -silent -sc
+scanme.sh {{binary}} -silent -td
+scanme.sh {{binary}} -silent -probe
+scanme.sh {{binary}} -silent -no-fallback
+scanme.sh {{binary}} -silent -cl
+scanme.sh {{binary}} -silent -server
+scanme.sh {{binary}} -silent -ip
+scanme.sh {{binary}} -silent -tls-grab
+scanme.sh {{binary}} -silent -unsafe
+scanme.sh {{binary}} -silent -x all
+scanme.sh {{binary}} -silent -body 'a=b'
+scanme.sh {{binary}} -silent -exclude-cdn
+scanme.sh {{binary}} -silent -ports https:443
+scanme.sh {{binary}} -silent -ztls
+https://scanme.sh?a=1*1 {{binary}} -silent

cmd/httpx/httpx.go

@@ -3,6 +3,8 @@ package main
 import (
 	"os"
 	"os/signal"
+	"runtime"
+	"runtime/pprof"
 
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/httpx/runner"
@@ -12,6 +14,24 @@ func main() {
 	// Parse the command line flags and read config files
 	options := runner.ParseOptions()
 
+	// Profiling related code
+	if options.Memprofile != "" {
+		f, err := os.Create(options.Memprofile)
+		if err != nil {
+			gologger.Fatal().Msgf("profile: could not create memory profile %q: %v", options.Memprofile, err)
+		}
+		old := runtime.MemProfileRate
+		runtime.MemProfileRate = 4096
+		gologger.Print().Msgf("profile: memory profiling enabled (rate %d), %s", runtime.MemProfileRate, options.Memprofile)
+
+		defer func() {
+			_ = pprof.Lookup("heap").WriteTo(f, 0)
+			f.Close()
+			runtime.MemProfileRate = old
+			gologger.Print().Msgf("profile: memory profiling disabled, %s", options.Memprofile)
+		}()
+	}
+
 	httpxRunner, err := runner.New(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)