From 38ad3c642dfbd3f25bef90f99cb5d0873956a04a Mon Sep 17 00:00:00 2001 From: emanuele Date: Mon, 18 Nov 2024 08:09:46 +0100 Subject: [PATCH] memory (#129) --- cffi_src/types.go | 66 +++++++++++++++++++------------------- client.go | 9 +++--- client_options.go | 52 +++++++++++++++--------------- connect.go | 19 +++++------ example/main.go | 44 ++++++++++++------------- jar.go | 2 +- profiles/profiles.go | 4 +-- roundtripper.go | 34 +++++++++++--------- tests/client_test_utils.go | 44 ++++++++++++------------- 9 files changed, 140 insertions(+), 134 deletions(-) diff --git a/cffi_src/types.go b/cffi_src/types.go index c6c6b72..bde44b7 100644 --- a/cffi_src/types.go +++ b/cffi_src/types.go @@ -32,9 +32,9 @@ type DestroyOutput struct { } type AddCookiesToSessionInput struct { - Cookies []Cookie `json:"cookies"` SessionId string `json:"sessionId"` Url string `json:"url"` + Cookies []Cookie `json:"cookies"` } type GetCookiesFromSessionInput struct { @@ -49,37 +49,37 @@ type CookiesFromSessionOutput struct { // RequestInput is the data a Python client can construct a client and request from. type RequestInput struct { - CatchPanics bool `json:"catchPanics"` CertificatePinningHosts map[string][]string `json:"certificatePinningHosts"` CustomTlsClient *CustomTlsClient `json:"customTlsClient"` TransportOptions *TransportOptions `json:"transportOptions"` - FollowRedirects bool `json:"followRedirects"` - ForceHttp1 bool `json:"forceHttp1"` - HeaderOrder []string `json:"headerOrder"` Headers map[string]string `json:"headers"` DefaultHeaders map[string][]string `json:"defaultHeaders"` ConnectHeaders map[string][]string `json:"connectHeaders"` - InsecureSkipVerify bool `json:"insecureSkipVerify"` - IsByteRequest bool `json:"isByteRequest"` - IsByteResponse bool `json:"isByteResponse"` - IsRotatingProxy bool `json:"isRotatingProxy"` - DisableIPV6 bool `json:"disableIPV6"` - DisableIPV4 bool `json:"disableIPV4"` LocalAddress *string `json:"localAddress"` ServerNameOverwrite *string `json:"serverNameOverwrite"` ProxyUrl *string `json:"proxyUrl"` RequestBody *string `json:"requestBody"` - RequestCookies []Cookie `json:"requestCookies"` - RequestMethod string `json:"requestMethod"` - RequestUrl string `json:"requestUrl"` RequestHostOverride *string `json:"requestHostOverride"` SessionId *string `json:"sessionId"` StreamOutputBlockSize *int `json:"streamOutputBlockSize"` StreamOutputEOFSymbol *string `json:"streamOutputEOFSymbol"` StreamOutputPath *string `json:"streamOutputPath"` + RequestMethod string `json:"requestMethod"` + RequestUrl string `json:"requestUrl"` + TLSClientIdentifier string `json:"tlsClientIdentifier"` + HeaderOrder []string `json:"headerOrder"` + RequestCookies []Cookie `json:"requestCookies"` TimeoutMilliseconds int `json:"timeoutMilliseconds"` TimeoutSeconds int `json:"timeoutSeconds"` - TLSClientIdentifier string `json:"tlsClientIdentifier"` + CatchPanics bool `json:"catchPanics"` + FollowRedirects bool `json:"followRedirects"` + ForceHttp1 bool `json:"forceHttp1"` + InsecureSkipVerify bool `json:"insecureSkipVerify"` + IsByteRequest bool `json:"isByteRequest"` + IsByteResponse bool `json:"isByteResponse"` + IsRotatingProxy bool `json:"isRotatingProxy"` + DisableIPV6 bool `json:"disableIPV6"` + DisableIPV4 bool `json:"disableIPV4"` WithDebug bool `json:"withDebug"` WithDefaultCookieJar bool `json:"withDefaultCookieJar"` WithoutCookieJar bool `json:"withoutCookieJar"` @@ -88,12 +88,11 @@ type RequestInput struct { // CustomTlsClient contains custom TLS specifications to construct a client from. type CustomTlsClient struct { - CertCompressionAlgo string `json:"certCompressionAlgo"` - ConnectionFlow uint32 `json:"connectionFlow"` H2Settings map[string]uint32 `json:"h2Settings"` - H2SettingsOrder []string `json:"h2SettingsOrder"` HeaderPriority *PriorityParam `json:"headerPriority"` + CertCompressionAlgo string `json:"certCompressionAlgo"` Ja3String string `json:"ja3String"` + H2SettingsOrder []string `json:"h2SettingsOrder"` KeyShareCurves []string `json:"keyShareCurves"` ALPNProtocols []string `json:"alpnProtocols"` ALPSProtocols []string `json:"alpsProtocols"` @@ -104,6 +103,7 @@ type CustomTlsClient struct { SupportedDelegatedCredentialsAlgorithms []string `json:"supportedDelegatedCredentialsAlgorithms"` SupportedSignatureAlgorithms []string `json:"supportedSignatureAlgorithms"` SupportedVersions []string `json:"supportedVersions"` + ConnectionFlow uint32 `json:"connectionFlow"` } type CandidateCipherSuites []CandidateCipherSuite @@ -127,17 +127,17 @@ type CandidateCipherSuite struct { // TransportOptions contains settings for the underlying http transport of the tls client type TransportOptions struct { - DisableKeepAlives bool `json:"disableKeepAlives"` - DisableCompression bool `json:"disableCompression"` - MaxIdleConns int `json:"maxIdleConns"` - MaxIdleConnsPerHost int `json:"maxIdleConnsPerHost"` - MaxConnsPerHost int `json:"maxConnsPerHost"` - MaxResponseHeaderBytes int64 `json:"maxResponseHeaderBytes"` // Zero means to use a default limit. - WriteBufferSize int `json:"writeBufferSize"` // If zero, a default (currently 4KB) is used. - ReadBufferSize int `json:"readBufferSize"` // If zero, a default (currently 4KB) is used. // IdleConnTimeout is the maximum amount of time an idle (keep-alive) // connection will remain idle before closing itself. Zero means no limit. - IdleConnTimeout *time.Duration `json:"idleConnTimeout"` + IdleConnTimeout *time.Duration `json:"idleConnTimeout"` + MaxIdleConns int `json:"maxIdleConns"` + MaxIdleConnsPerHost int `json:"maxIdleConnsPerHost"` + MaxConnsPerHost int `json:"maxConnsPerHost"` + MaxResponseHeaderBytes int64 `json:"maxResponseHeaderBytes"` // Zero means to use a default limit. + WriteBufferSize int `json:"writeBufferSize"` // If zero, a default (currently 4KB) is used. + ReadBufferSize int `json:"readBufferSize"` // If zero, a default (currently 4KB) is used. + DisableKeepAlives bool `json:"disableKeepAlives"` + DisableCompression bool `json:"disableCompression"` } type PriorityFrames struct { @@ -146,18 +146,18 @@ type PriorityFrames struct { } type PriorityParam struct { - Exclusive bool `json:"exclusive"` StreamDep uint32 `json:"streamDep"` + Exclusive bool `json:"exclusive"` Weight uint8 `json:"weight"` } type Cookie struct { - Domain string `json:"domain"` Expires Timestamp `json:"expires"` - MaxAge int `json:"maxAge"` + Domain string `json:"domain"` Name string `json:"name"` Path string `json:"path"` Value string `json:"value"` + MaxAge int `json:"maxAge"` } type Timestamp struct { @@ -183,12 +183,12 @@ func (p *Timestamp) MarshalJSON() ([]byte, error) { // Response is the response that is sent back to the Python client. type Response struct { - Id string `json:"id"` - Body string `json:"body"` Cookies map[string]string `json:"cookies"` Headers map[string][]string `json:"headers"` + Id string `json:"id"` + Body string `json:"body"` SessionId string `json:"sessionId,omitempty"` - Status int `json:"status"` Target string `json:"target"` UsedProtocol string `json:"usedProtocol"` + Status int `json:"status"` } diff --git a/client.go b/client.go index f1aeed1..4e5af7f 100644 --- a/client.go +++ b/client.go @@ -43,12 +43,13 @@ type HttpClient interface { var _ HttpClient = (*httpClient)(nil) type httpClient struct { - http.Client - headerLck sync.Mutex - logger Logger - config *httpClientConfig + logger Logger bandwidthTracker bandwidth.BandwidthTracker + config *httpClientConfig + + http.Client + headerLck sync.Mutex } var DefaultTimeoutSeconds = 30 diff --git a/client_options.go b/client_options.go index 8aa3fb1..28f9726 100644 --- a/client_options.go +++ b/client_options.go @@ -14,53 +14,55 @@ import ( type HttpClientOption func(config *httpClientConfig) type TransportOptions struct { - DisableKeepAlives bool - DisableCompression bool + // KeyLogWriter is an io.Writer that the TLS client will use to write the + // TLS master secrets to. This can be used to decrypt TLS connections in + // Wireshark and other applications. + KeyLogWriter io.Writer + // IdleConnTimeout is the maximum amount of time an idle (keep-alive) + // connection will remain idle before closing itself. Zero means no limit. + IdleConnTimeout *time.Duration + // RootCAs is the set of root certificate authorities used to verify + // the remote server's certificate. + RootCAs *x509.CertPool MaxIdleConns int MaxIdleConnsPerHost int MaxConnsPerHost int MaxResponseHeaderBytes int64 // Zero means to use a default limit. WriteBufferSize int // If zero, a default (currently 4KB) is used. ReadBufferSize int // If zero, a default (currently 4KB) is used. - // IdleConnTimeout is the maximum amount of time an idle (keep-alive) - // connection will remain idle before closing itself. Zero means no limit. - IdleConnTimeout *time.Duration - // RootCAs is the set of root certificate authorities used to verify - // the remote server's certificate. - RootCAs *x509.CertPool - // KeyLogWriter is an io.Writer that the TLS client will use to write the - // TLS master secrets to. This can be used to decrypt TLS connections in - // Wireshark and other applications. - KeyLogWriter io.Writer + DisableKeepAlives bool + DisableCompression bool } type BadPinHandlerFunc func(req *http.Request) type httpClientConfig struct { + cookieJar http.CookieJar + customRedirectFunc func(req *http.Request, via []*http.Request) error + certificatePins map[string][]string + defaultHeaders http.Header + connectHeaders http.Header + badPinHandler BadPinHandlerFunc + transportOptions *TransportOptions + localAddr *net.TCPAddr + + dialer net.Dialer + + proxyUrl string + serverNameOverwrite string + clientProfile profiles.ClientProfile + timeout time.Duration catchPanics bool debug bool followRedirects bool - customRedirectFunc func(req *http.Request, via []*http.Request) error insecureSkipVerify bool - certificatePins map[string][]string - defaultHeaders http.Header - connectHeaders http.Header - badPinHandler BadPinHandlerFunc - proxyUrl string - serverNameOverwrite string - transportOptions *TransportOptions - cookieJar http.CookieJar - clientProfile profiles.ClientProfile withRandomTlsExtensionOrder bool forceHttp1 bool - timeout time.Duration - localAddr *net.TCPAddr // Establish a connection to origin server via ipv4 only disableIPV6 bool // Establish a connection to origin server via ipv6 only disableIPV4 bool - dialer net.Dialer enabledBandwidthTracker bool } diff --git a/connect.go b/connect.go index 0718cdc..6f95232 100644 --- a/connect.go +++ b/connect.go @@ -75,20 +75,21 @@ func (s *socksContextDialer) DialContext(ctx context.Context, network, address s // connectDialer allows to configure one-time use HTTP CONNECT client type connectDialer struct { - logger Logger - ProxyUrl url.URL - DefaultHeader http.Header - - Dialer net.Dialer // overridden dialer allow to control establishment of TCP connection + logger Logger + cachedH2RawConn net.Conn + DefaultHeader http.Header // overridden DialTLS allows user to control establishment of TLS connection // MUST return connection with completed Handshake, and NegotiatedProtocol DialTLS func(network string, address string) (net.Conn, string, error) - Timeout time.Duration - EnableH2ConnReuse bool - cacheH2Mu sync.Mutex cachedH2ClientConn *http2.ClientConn - cachedH2RawConn net.Conn + ProxyUrl url.URL + + Dialer net.Dialer // overridden dialer allow to control establishment of TCP connection + + Timeout time.Duration + cacheH2Mu sync.Mutex + EnableH2ConnReuse bool } // newConnectDialer creates a dialer to issue CONNECT requests and tunnel traffic via HTTP/S proxy. diff --git a/example/main.go b/example/main.go index 15f19a3..f7d98ce 100644 --- a/example/main.go +++ b/example/main.go @@ -35,45 +35,42 @@ type TlsApiResponse struct { HTTPVersion string `json:"http_version"` Method string `json:"method"` TLS struct { - Ciphers []string `json:"ciphers"` - Extensions []struct { + TLSVersionRecord string `json:"tls_version_record"` + TLSVersionNegotiated string `json:"tls_version_negotiated"` + Ja3 string `json:"ja3"` + Ja3Hash string `json:"ja3_hash"` + ClientRandom string `json:"client_random"` + SessionID string `json:"session_id"` + Ciphers []string `json:"ciphers"` + Extensions []struct { + EllipticCurvesPointFormats interface{} `json:"elliptic_curves_point_formats,omitempty"` Name string `json:"name"` ServerName string `json:"server_name,omitempty"` Data string `json:"data,omitempty"` + PskKeyExchangeMode string `json:"PSK_Key_Exchange_Mode,omitempty"` SupportedGroups []string `json:"supported_groups,omitempty"` - EllipticCurvesPointFormats interface{} `json:"elliptic_curves_point_formats,omitempty"` Protocols []string `json:"protocols,omitempty"` - StatusRequest struct { + SignatureAlgorithms []string `json:"signature_algorithms,omitempty"` + SharedKeys []struct { + TLSGrease0X7A7A string `json:"TLS_GREASE (0x7a7a),omitempty"` + X2551929 string `json:"X25519 (29),omitempty"` + } `json:"shared_keys,omitempty"` + Versions []string `json:"versions,omitempty"` + Algorithms []string `json:"algorithms,omitempty"` + StatusRequest struct { CertificateStatusType string `json:"certificate_status_type"` ResponderIDListLength int `json:"responder_id_list_length"` RequestExtensionsLength int `json:"request_extensions_length"` } `json:"status_request,omitempty"` - SignatureAlgorithms []string `json:"signature_algorithms,omitempty"` - SharedKeys []struct { - TLSGrease0X7A7A string `json:"TLS_GREASE (0x7a7a),omitempty"` - X2551929 string `json:"X25519 (29),omitempty"` - } `json:"shared_keys,omitempty"` - PskKeyExchangeMode string `json:"PSK_Key_Exchange_Mode,omitempty"` - Versions []string `json:"versions,omitempty"` - Algorithms []string `json:"algorithms,omitempty"` - PaddingDataLength int `json:"padding_data_length,omitempty"` + PaddingDataLength int `json:"padding_data_length,omitempty"` } `json:"extensions"` - TLSVersionRecord string `json:"tls_version_record"` - TLSVersionNegotiated string `json:"tls_version_negotiated"` - Ja3 string `json:"ja3"` - Ja3Hash string `json:"ja3_hash"` - ClientRandom string `json:"client_random"` - SessionID string `json:"session_id"` } `json:"tls"` HTTP2 struct { AkamaiFingerprint string `json:"akamai_fingerprint"` AkamaiFingerprintHash string `json:"akamai_fingerprint_hash"` SentFrames []struct { FrameType string `json:"frame_type"` - Length int `json:"length"` Settings []string `json:"settings,omitempty"` - Increment int `json:"increment,omitempty"` - StreamID int `json:"stream_id,omitempty"` Headers []string `json:"headers,omitempty"` Flags []string `json:"flags,omitempty"` Priority struct { @@ -81,6 +78,9 @@ type TlsApiResponse struct { DependsOn int `json:"depends_on"` Exclusive int `json:"exclusive"` } `json:"priority,omitempty"` + Length int `json:"length"` + Increment int `json:"increment,omitempty"` + StreamID int `json:"stream_id,omitempty"` } `json:"sent_frames"` } `json:"http2"` HTTP1 struct { diff --git a/jar.go b/jar.go index 822f06d..605058b 100644 --- a/jar.go +++ b/jar.go @@ -13,10 +13,10 @@ import ( type CookieJarOption func(config *cookieJarConfig) type cookieJarConfig struct { + logger Logger skipExisting bool debug bool allowEmptyCookies bool - logger Logger } func WithSkipExisting() CookieJarOption { diff --git a/profiles/profiles.go b/profiles/profiles.go index c025f3b..0e86860 100644 --- a/profiles/profiles.go +++ b/profiles/profiles.go @@ -71,12 +71,12 @@ var MappedTLSClients = map[string]ClientProfile{ type ClientProfile struct { clientHelloId tls.ClientHelloID - connectionFlow uint32 headerPriority *http2.PriorityParam + settings map[http2.SettingID]uint32 priorities []http2.Priority pseudoHeaderOrder []string - settings map[http2.SettingID]uint32 settingsOrder []http2.SettingID + connectionFlow uint32 } func NewClientProfile(clientHelloId tls.ClientHelloID, settings map[http2.SettingID]uint32, settingsOrder []http2.SettingID, pseudoHeaderOrder []string, connectionFlow uint32, priorities []http2.Priority, headerPriority *http2.PriorityParam) ClientProfile { diff --git a/roundtripper.go b/roundtripper.go index 479a2c7..1d50aea 100644 --- a/roundtripper.go +++ b/roundtripper.go @@ -22,32 +22,34 @@ const defaultIdleConnectionTimeout = 90 * time.Second var errProtocolNegotiated = errors.New("protocol negotiated") type roundTripper struct { - sync.Mutex + clientHelloId tls.ClientHelloID + certificatePinner CertificatePinner + + dialer proxy.ContextDialer + + bandwidthTracker bandwidth.BandwidthTracker + + clientSessionCache tls.ClientSessionCache + badPinHandlerFunc BadPinHandlerFunc cachedConnections map[string]net.Conn cachedTransports map[string]http.RoundTripper + headerPriority *http2.PriorityParam + settings map[http2.SettingID]uint32 + transportOptions *TransportOptions + serverNameOverwrite string + priorities []http2.Priority + pseudoHeaderOrder []string + settingsOrder []http2.SettingID + sync.Mutex + cachedTransportsLck sync.Mutex - certificatePinner CertificatePinner - clientHelloId tls.ClientHelloID connectionFlow uint32 - dialer proxy.ContextDialer - forceHttp1 bool - bandwidthTracker bandwidth.BandwidthTracker - - headerPriority *http2.PriorityParam - clientSessionCache tls.ClientSessionCache - insecureSkipVerify bool - priorities []http2.Priority - pseudoHeaderOrder []string - serverNameOverwrite string - settings map[http2.SettingID]uint32 - settingsOrder []http2.SettingID - transportOptions *TransportOptions withRandomTlsExtensionOrder bool disableIPV6 bool disableIPV4 bool diff --git a/tests/client_test_utils.go b/tests/client_test_utils.go index 2633564..d4ecef1 100644 --- a/tests/client_test_utils.go +++ b/tests/client_test_utils.go @@ -10,45 +10,42 @@ type TlsApiResponse struct { HTTPVersion string `json:"http_version"` Method string `json:"method"` TLS struct { - Ciphers []string `json:"ciphers"` - Extensions []struct { + TLSVersionRecord string `json:"tls_version_record"` + TLSVersionNegotiated string `json:"tls_version_negotiated"` + Ja3 string `json:"ja3"` + Ja3Hash string `json:"ja3_hash"` + ClientRandom string `json:"client_random"` + SessionID string `json:"session_id"` + Ciphers []string `json:"ciphers"` + Extensions []struct { + EllipticCurvesPointFormats interface{} `json:"elliptic_curves_point_formats,omitempty"` Name string `json:"name"` ServerName string `json:"server_name,omitempty"` Data string `json:"data,omitempty"` + PskKeyExchangeMode string `json:"PSK_Key_Exchange_Mode,omitempty"` SupportedGroups []string `json:"supported_groups,omitempty"` - EllipticCurvesPointFormats interface{} `json:"elliptic_curves_point_formats,omitempty"` Protocols []string `json:"protocols,omitempty"` - StatusRequest struct { + SignatureAlgorithms []string `json:"signature_algorithms,omitempty"` + SharedKeys []struct { + TLSGrease0X7A7A string `json:"TLS_GREASE (0x7a7a),omitempty"` + X2551929 string `json:"X25519 (29),omitempty"` + } `json:"shared_keys,omitempty"` + Versions []string `json:"versions,omitempty"` + Algorithms []string `json:"algorithms,omitempty"` + StatusRequest struct { CertificateStatusType string `json:"certificate_status_type"` ResponderIDListLength int `json:"responder_id_list_length"` RequestExtensionsLength int `json:"request_extensions_length"` } `json:"status_request,omitempty"` - SignatureAlgorithms []string `json:"signature_algorithms,omitempty"` - SharedKeys []struct { - TLSGrease0X7A7A string `json:"TLS_GREASE (0x7a7a),omitempty"` - X2551929 string `json:"X25519 (29),omitempty"` - } `json:"shared_keys,omitempty"` - PskKeyExchangeMode string `json:"PSK_Key_Exchange_Mode,omitempty"` - Versions []string `json:"versions,omitempty"` - Algorithms []string `json:"algorithms,omitempty"` - PaddingDataLength int `json:"padding_data_length,omitempty"` + PaddingDataLength int `json:"padding_data_length,omitempty"` } `json:"extensions"` - TLSVersionRecord string `json:"tls_version_record"` - TLSVersionNegotiated string `json:"tls_version_negotiated"` - Ja3 string `json:"ja3"` - Ja3Hash string `json:"ja3_hash"` - ClientRandom string `json:"client_random"` - SessionID string `json:"session_id"` } `json:"tls"` HTTP2 struct { AkamaiFingerprint string `json:"akamai_fingerprint"` AkamaiFingerprintHash string `json:"akamai_fingerprint_hash"` SentFrames []struct { FrameType string `json:"frame_type"` - Length int `json:"length"` Settings []string `json:"settings,omitempty"` - Increment int `json:"increment,omitempty"` - StreamID int `json:"stream_id,omitempty"` Headers []string `json:"headers,omitempty"` Flags []string `json:"flags,omitempty"` Priority struct { @@ -56,6 +53,9 @@ type TlsApiResponse struct { DependsOn int `json:"depends_on"` Exclusive int `json:"exclusive"` } `json:"priority,omitempty"` + Length int `json:"length"` + Increment int `json:"increment,omitempty"` + StreamID int `json:"stream_id,omitempty"` } `json:"sent_frames"` } `json:"http2"` HTTP1 struct {