Merge pull request #105 from ulixee/test-headers

fix(mitm): use user agent for go connect
ulixee · Aug 30, 2024 · 3a0f1c1 · 3a0f1c1
2 parents 0c27903 + 21680d3
commit 3a0f1c1
Show file tree

Hide file tree

Showing 16 changed files with 47 additions and 29 deletions.
diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml
@@ -57,7 +57,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.23'
 
       - name: Add msbuild to PATH
         uses: microsoft/setup-msbuild@v2

diff --git a/.github/workflows/new-profiles.yml b/.github/workflows/new-profiles.yml
@@ -43,6 +43,10 @@ jobs:
           cache: 'yarn'
           cache-dependency-path: unblocked/yarn.lock
 
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+
       - name: Install ulixee/shared
         run: yarn build
         working-directory: ./shared

diff --git a/.github/workflows/new-version.yml b/.github/workflows/new-version.yml
@@ -16,7 +16,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.23'
 
       - name: Build Sockets
         uses: goreleaser/goreleaser-action@v5

diff --git a/.github/workflows/profile-doms.yml b/.github/workflows/profile-doms.yml
@@ -39,6 +39,10 @@ jobs:
           cache: 'yarn'
           cache-dependency-path: unblocked/yarn.lock
 
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.23'
+
       - name: Install ulixee/shared
         run: yarn build
         working-directory: ./shared

diff --git a/agent/mitm-socket/go/connect.go b/agent/mitm-socket/go/connect.go
@@ -200,6 +200,7 @@ type ConnectArgs struct {
 type SessionArgs struct {
 	IpcSocketPath      string
 	RejectUnauthorized bool
+	UserAgent           string
 	ClientHelloId      string
 	TcpTtl             int
 	TcpWindowSize      int

diff --git a/agent/mitm-socket/go/dialer.go b/agent/mitm-socket/go/dialer.go
@@ -25,7 +25,7 @@ func Dial(addr string, connectArgs ConnectArgs, sessionArgs SessionArgs) (net.Co
 			return DialAddrViaSock5Proxy(dialer, addr, proxyUrl)
 		}
 
-		return DialAddrViaHttpProxy(dialer, addr, proxyUrl, !sessionArgs.RejectUnauthorized)
+		return DialAddrViaHttpProxy(dialer, addr, proxyUrl, !sessionArgs.RejectUnauthorized, sessionArgs.UserAgent)
 	}
 
 	dialConn, err := dialer.Dial("tcp", addr)

diff --git a/agent/mitm-socket/go/dialer_proxy_http.go b/agent/mitm-socket/go/dialer_proxy_http.go
@@ -15,7 +15,7 @@ import (
 	"strings"
 )
 
-func DialAddrViaHttpProxy(dialer net.Dialer, addr string, proxyUrl *url.URL, allowInsecure bool) (net.Conn, error) {
+func DialAddrViaHttpProxy(dialer net.Dialer, addr string, proxyUrl *url.URL, allowInsecure bool, userAgent string) (net.Conn, error) {
 	isSecure, proxyHost, err := getCleanHost(proxyUrl)
 
 	fmt.Printf("Dialing proxy connect %s to %s\n", proxyHost, addr)
@@ -25,6 +25,9 @@ func DialAddrViaHttpProxy(dialer net.Dialer, addr string, proxyUrl *url.URL, all
 		Host:   addr,
 		Header: make(http.Header),
 	}
+    if userAgent != "" {
+	    connectReq.Header.Set("User-Agent", userAgent)
+	}
 
 	if proxyUrl.User != nil {
 		authBuffer := bytes.NewBuffer(nil)

diff --git a/agent/mitm-socket/go/go.mod b/agent/mitm-socket/go/go.mod
@@ -1,18 +1,18 @@
 module connect
 
-go 1.20
+go 1.23
 
 require (
-	github.com/Microsoft/go-winio v0.5.1
+	github.com/Microsoft/go-winio v0.6.2
 	github.com/refraction-networking/utls v1.6.7
-	golang.org/x/net v0.23.0
+	golang.org/x/net v0.28.0
 )
 
 require (
 	github.com/andybalholm/brotli v1.0.6 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/klauspost/compress v1.17.4 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/crypto v0.26.0 // indirect
+	golang.org/x/sys v0.23.0 // indirect
+	golang.org/x/text v0.17.0 // indirect
 )
diff --git a/agent/mitm-socket/go/go.sum b/agent/mitm-socket/go/go.sum
@@ -1,25 +1,18 @@
-github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
-github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
 github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
 github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4=
 github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
 github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
-github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
+golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
+golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
diff --git a/agent/mitm-socket/lib/BaseIpcHandler.ts b/agent/mitm-socket/lib/BaseIpcHandler.ts
@@ -220,6 +220,7 @@ export default abstract class BaseIpcHandler {
 export interface IGoIpcOpts {
   mode?: 'certs' | 'proxy';
   storageDir?: string;
+  userAgent?: string;
   ipcSocketPath?: string;
   clientHelloId?: string;
   tcpTtl?: number;

diff --git a/agent/mitm-socket/test/proxy.test.ts b/agent/mitm-socket/test/proxy.test.ts
@@ -10,6 +10,7 @@ import WebSocket = require('ws');
 import * as socks5 from 'simple-socks';
 import { createPromise } from '@ulixee/commons/lib/utils';
 import * as http2 from 'http2';
+import exp = require('node:constants');
 import MitmSocket from '../index';
 import MitmSocketSession from '../lib/MitmSocketSession';
 
@@ -26,6 +27,7 @@ beforeAll(() => {
   mitmSocketSession = new MitmSocketSession(TestLogger.forTest(module), {
     clientHelloId: 'chrome-114',
     rejectUnauthorized: false,
+    userAgent: 'Chrome',
   });
   Helpers.onClose(() => mitmSocketSession.close(), true);
 });
@@ -51,6 +53,9 @@ test('should be able to send a request through a proxy', async () => {
   const httpResponse = await httpGetWithSocket(`${server.baseUrl}/any`, {}, tlsConnection.socket);
   expect(httpResponse).toBe(htmlString);
   expect(connect).toHaveBeenCalledTimes(1);
+  expect(connect.mock.calls[0][0].url).toBe(`localhost:${server.port}`);
+  console.log(connect.mock.calls[0][0].headers);
+  expect(connect.mock.calls[0][0].headers['user-agent']).toBe('Chrome');
 });
 
 test('should be able to send a request through a secure proxy with auth', async () => {

diff --git a/agent/mitm/lib/MitmRequestAgent.ts b/agent/mitm/lib/MitmRequestAgent.ts
@@ -47,6 +47,7 @@ export default class MitmRequestAgent {
       tcpTtl: tcpSettings?.tcpTtl,
       tcpWindowSize: tcpSettings?.tcpWindowSize,
       debug: env.isDebug,
+      userAgent: tlsSettings.proxyUseragent,
     });
     this.maxConnectionsPerOrigin =
       tlsSettings?.socketsPerOrigin ?? MitmRequestAgent.defaultMaxConnectionsPerOrigin;

diff --git a/browser-profiler/main/scripts/runBrowserstack.ts b/browser-profiler/main/scripts/runBrowserstack.ts
@@ -34,7 +34,8 @@ export default async function runBrowserstack(): Promise<void> {
 
     // too old for the double agent suite
     if (browser.name === 'Firefox' && Number(browser.version.major) < 63) continue;
-    if (browser.name === 'Chrome' && Number(browser.version.major) < 58) continue;
+    // TODO: The dom environment plugin hangs <= 94. Need to investigate for scraper report eventually
+    if (browser.name === 'Chrome' && Number(browser.version.major) < 95) continue;
     if (browser.name === 'Safari' && Number(browser.version.major) < 11) continue;
     // no support for Promises, lambdas... detections need refactor for support
     if (browser.name === 'IE') continue;

diff --git a/browser-profiler/main/scripts/runBrowserstackDoms.ts b/browser-profiler/main/scripts/runBrowserstackDoms.ts
@@ -39,6 +39,9 @@ export default async function runBrowserstack(): Promise<void> {
       continue;
     }
 
+    // TODO: The dom environment plugin hangs <= 94. Need to investigate for scraper report eventually
+    if (browser.name === 'Chrome' && Number(browser.version.major) < 95) continue;
+
     // 1. Does this need to run? Clean up as needed.
     const domDir = Path.join(baseDomsDir, `${userAgentId}--${features}`);
     if (await existsAsync(domDir)) {

diff --git a/plugins/default-browser-emulator/lib/helpers/configureSessionTls.ts b/plugins/default-browser-emulator/lib/helpers/configureSessionTls.ts
@@ -5,6 +5,7 @@ export default function configureSessionTls(
   emulationProfile: IEmulationProfile,
   settings: ITlsSettings,
 ): void {
-  const { browserName, browserVersion } = emulationProfile.userAgentOption;
+  const { browserName, browserVersion, string } = emulationProfile.userAgentOption;
   settings.tlsClientHelloId = `${browserName}-${browserVersion.major}`;
+  settings.proxyUseragent = string;
 }
diff --git a/specification/agent/net/ITlsSettings.ts b/specification/agent/net/ITlsSettings.ts
@@ -1,4 +1,5 @@
 export default interface ITlsSettings {
   tlsClientHelloId?: string;
   socketsPerOrigin?: number;
+  proxyUseragent?: string;
 }