Skip to content

Commit

Permalink
memory (#129)
Browse files Browse the repository at this point in the history
  • Loading branch information
@ManuCiao10
ManuCiao10 authored Nov 18, 2024
1 parent 3c7ca0a commit 38ad3c6
Show file tree
Hide file tree
Showing 9 changed files with 140 additions and 134 deletions.
66 changes: 33 additions & 33 deletions cffi_src/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ type DestroyOutput struct {
}

type AddCookiesToSessionInput struct {
Cookies []Cookie `json:"cookies"`
SessionId string `json:"sessionId"`
Url string `json:"url"`
Cookies []Cookie `json:"cookies"`
}

type GetCookiesFromSessionInput struct {
Expand All @@ -49,37 +49,37 @@ type CookiesFromSessionOutput struct {

// RequestInput is the data a Python client can construct a client and request from.
type RequestInput struct {
CatchPanics bool `json:"catchPanics"`
CertificatePinningHosts map[string][]string `json:"certificatePinningHosts"`
CustomTlsClient *CustomTlsClient `json:"customTlsClient"`
TransportOptions *TransportOptions `json:"transportOptions"`
FollowRedirects bool `json:"followRedirects"`
ForceHttp1 bool `json:"forceHttp1"`
HeaderOrder []string `json:"headerOrder"`
Headers map[string]string `json:"headers"`
DefaultHeaders map[string][]string `json:"defaultHeaders"`
ConnectHeaders map[string][]string `json:"connectHeaders"`
InsecureSkipVerify bool `json:"insecureSkipVerify"`
IsByteRequest bool `json:"isByteRequest"`
IsByteResponse bool `json:"isByteResponse"`
IsRotatingProxy bool `json:"isRotatingProxy"`
DisableIPV6 bool `json:"disableIPV6"`
DisableIPV4 bool `json:"disableIPV4"`
LocalAddress *string `json:"localAddress"`
ServerNameOverwrite *string `json:"serverNameOverwrite"`
ProxyUrl *string `json:"proxyUrl"`
RequestBody *string `json:"requestBody"`
RequestCookies []Cookie `json:"requestCookies"`
RequestMethod string `json:"requestMethod"`
RequestUrl string `json:"requestUrl"`
RequestHostOverride *string `json:"requestHostOverride"`
SessionId *string `json:"sessionId"`
StreamOutputBlockSize *int `json:"streamOutputBlockSize"`
StreamOutputEOFSymbol *string `json:"streamOutputEOFSymbol"`
StreamOutputPath *string `json:"streamOutputPath"`
RequestMethod string `json:"requestMethod"`
RequestUrl string `json:"requestUrl"`
TLSClientIdentifier string `json:"tlsClientIdentifier"`
HeaderOrder []string `json:"headerOrder"`
RequestCookies []Cookie `json:"requestCookies"`
TimeoutMilliseconds int `json:"timeoutMilliseconds"`
TimeoutSeconds int `json:"timeoutSeconds"`
TLSClientIdentifier string `json:"tlsClientIdentifier"`
CatchPanics bool `json:"catchPanics"`
FollowRedirects bool `json:"followRedirects"`
ForceHttp1 bool `json:"forceHttp1"`
InsecureSkipVerify bool `json:"insecureSkipVerify"`
IsByteRequest bool `json:"isByteRequest"`
IsByteResponse bool `json:"isByteResponse"`
IsRotatingProxy bool `json:"isRotatingProxy"`
DisableIPV6 bool `json:"disableIPV6"`
DisableIPV4 bool `json:"disableIPV4"`
WithDebug bool `json:"withDebug"`
WithDefaultCookieJar bool `json:"withDefaultCookieJar"`
WithoutCookieJar bool `json:"withoutCookieJar"`
Expand All @@ -88,12 +88,11 @@ type RequestInput struct {

// CustomTlsClient contains custom TLS specifications to construct a client from.
type CustomTlsClient struct {
CertCompressionAlgo string `json:"certCompressionAlgo"`
ConnectionFlow uint32 `json:"connectionFlow"`
H2Settings map[string]uint32 `json:"h2Settings"`
H2SettingsOrder []string `json:"h2SettingsOrder"`
HeaderPriority *PriorityParam `json:"headerPriority"`
CertCompressionAlgo string `json:"certCompressionAlgo"`
Ja3String string `json:"ja3String"`
H2SettingsOrder []string `json:"h2SettingsOrder"`
KeyShareCurves []string `json:"keyShareCurves"`
ALPNProtocols []string `json:"alpnProtocols"`
ALPSProtocols []string `json:"alpsProtocols"`
Expand All @@ -104,6 +103,7 @@ type CustomTlsClient struct {
SupportedDelegatedCredentialsAlgorithms []string `json:"supportedDelegatedCredentialsAlgorithms"`
SupportedSignatureAlgorithms []string `json:"supportedSignatureAlgorithms"`
SupportedVersions []string `json:"supportedVersions"`
ConnectionFlow uint32 `json:"connectionFlow"`
}

type CandidateCipherSuites []CandidateCipherSuite
Expand All @@ -127,17 +127,17 @@ type CandidateCipherSuite struct {

// TransportOptions contains settings for the underlying http transport of the tls client
type TransportOptions struct {
DisableKeepAlives bool `json:"disableKeepAlives"`
DisableCompression bool `json:"disableCompression"`
MaxIdleConns int `json:"maxIdleConns"`
MaxIdleConnsPerHost int `json:"maxIdleConnsPerHost"`
MaxConnsPerHost int `json:"maxConnsPerHost"`
MaxResponseHeaderBytes int64 `json:"maxResponseHeaderBytes"` // Zero means to use a default limit.
WriteBufferSize int `json:"writeBufferSize"` // If zero, a default (currently 4KB) is used.
ReadBufferSize int `json:"readBufferSize"` // If zero, a default (currently 4KB) is used.
// IdleConnTimeout is the maximum amount of time an idle (keep-alive)
// connection will remain idle before closing itself. Zero means no limit.
IdleConnTimeout *time.Duration `json:"idleConnTimeout"`
IdleConnTimeout *time.Duration `json:"idleConnTimeout"`
MaxIdleConns int `json:"maxIdleConns"`
MaxIdleConnsPerHost int `json:"maxIdleConnsPerHost"`
MaxConnsPerHost int `json:"maxConnsPerHost"`
MaxResponseHeaderBytes int64 `json:"maxResponseHeaderBytes"` // Zero means to use a default limit.
WriteBufferSize int `json:"writeBufferSize"` // If zero, a default (currently 4KB) is used.
ReadBufferSize int `json:"readBufferSize"` // If zero, a default (currently 4KB) is used.
DisableKeepAlives bool `json:"disableKeepAlives"`
DisableCompression bool `json:"disableCompression"`
}

type PriorityFrames struct {
Expand All @@ -146,18 +146,18 @@ type PriorityFrames struct {
}

type PriorityParam struct {
Exclusive bool `json:"exclusive"`
StreamDep uint32 `json:"streamDep"`
Exclusive bool `json:"exclusive"`
Weight uint8 `json:"weight"`
}

type Cookie struct {
Domain string `json:"domain"`
Expires Timestamp `json:"expires"`
MaxAge int `json:"maxAge"`
Domain string `json:"domain"`
Name string `json:"name"`
Path string `json:"path"`
Value string `json:"value"`
MaxAge int `json:"maxAge"`
}

type Timestamp struct {
Expand All @@ -183,12 +183,12 @@ func (p *Timestamp) MarshalJSON() ([]byte, error) {

// Response is the response that is sent back to the Python client.
type Response struct {
Id string `json:"id"`
Body string `json:"body"`
Cookies map[string]string `json:"cookies"`
Headers map[string][]string `json:"headers"`
Id string `json:"id"`
Body string `json:"body"`
SessionId string `json:"sessionId,omitempty"`
Status int `json:"status"`
Target string `json:"target"`
UsedProtocol string `json:"usedProtocol"`
Status int `json:"status"`
}
9 changes: 5 additions & 4 deletions client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,13 @@ type HttpClient interface {
var _ HttpClient = (*httpClient)(nil)

type httpClient struct {
http.Client
headerLck sync.Mutex
logger Logger
config *httpClientConfig
logger Logger

bandwidthTracker bandwidth.BandwidthTracker
config *httpClientConfig

http.Client
headerLck sync.Mutex
}

var DefaultTimeoutSeconds = 30
Expand Down
52 changes: 27 additions & 25 deletions client_options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,53 +14,55 @@ import (
type HttpClientOption func(config *httpClientConfig)

type TransportOptions struct {
DisableKeepAlives bool
DisableCompression bool
// KeyLogWriter is an io.Writer that the TLS client will use to write the
// TLS master secrets to. This can be used to decrypt TLS connections in
// Wireshark and other applications.
KeyLogWriter io.Writer
// IdleConnTimeout is the maximum amount of time an idle (keep-alive)
// connection will remain idle before closing itself. Zero means no limit.
IdleConnTimeout *time.Duration
// RootCAs is the set of root certificate authorities used to verify
// the remote server's certificate.
RootCAs *x509.CertPool
MaxIdleConns int
MaxIdleConnsPerHost int
MaxConnsPerHost int
MaxResponseHeaderBytes int64 // Zero means to use a default limit.
WriteBufferSize int // If zero, a default (currently 4KB) is used.
ReadBufferSize int // If zero, a default (currently 4KB) is used.
// IdleConnTimeout is the maximum amount of time an idle (keep-alive)
// connection will remain idle before closing itself. Zero means no limit.
IdleConnTimeout *time.Duration
// RootCAs is the set of root certificate authorities used to verify
// the remote server's certificate.
RootCAs *x509.CertPool
// KeyLogWriter is an io.Writer that the TLS client will use to write the
// TLS master secrets to. This can be used to decrypt TLS connections in
// Wireshark and other applications.
KeyLogWriter io.Writer
DisableKeepAlives bool
DisableCompression bool
}

type BadPinHandlerFunc func(req *http.Request)

type httpClientConfig struct {
cookieJar http.CookieJar
customRedirectFunc func(req *http.Request, via []*http.Request) error
certificatePins map[string][]string
defaultHeaders http.Header
connectHeaders http.Header
badPinHandler BadPinHandlerFunc
transportOptions *TransportOptions
localAddr *net.TCPAddr

dialer net.Dialer

proxyUrl string
serverNameOverwrite string
clientProfile profiles.ClientProfile
timeout time.Duration
catchPanics bool
debug bool
followRedirects bool
customRedirectFunc func(req *http.Request, via []*http.Request) error
insecureSkipVerify bool
certificatePins map[string][]string
defaultHeaders http.Header
connectHeaders http.Header
badPinHandler BadPinHandlerFunc
proxyUrl string
serverNameOverwrite string
transportOptions *TransportOptions
cookieJar http.CookieJar
clientProfile profiles.ClientProfile
withRandomTlsExtensionOrder bool
forceHttp1 bool
timeout time.Duration
localAddr *net.TCPAddr

// Establish a connection to origin server via ipv4 only
disableIPV6 bool
// Establish a connection to origin server via ipv6 only
disableIPV4 bool
dialer net.Dialer

enabledBandwidthTracker bool
}
Expand Down
19 changes: 10 additions & 9 deletions connect.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,20 +75,21 @@ func (s *socksContextDialer) DialContext(ctx context.Context, network, address s

// connectDialer allows to configure one-time use HTTP CONNECT client
type connectDialer struct {
logger Logger
ProxyUrl url.URL
DefaultHeader http.Header

Dialer net.Dialer // overridden dialer allow to control establishment of TCP connection
logger Logger
cachedH2RawConn net.Conn
DefaultHeader http.Header

// overridden DialTLS allows user to control establishment of TLS connection
// MUST return connection with completed Handshake, and NegotiatedProtocol
DialTLS func(network string, address string) (net.Conn, string, error)
Timeout time.Duration
EnableH2ConnReuse bool
cacheH2Mu sync.Mutex
cachedH2ClientConn *http2.ClientConn
cachedH2RawConn net.Conn
ProxyUrl url.URL

Dialer net.Dialer // overridden dialer allow to control establishment of TCP connection

Timeout time.Duration
cacheH2Mu sync.Mutex
EnableH2ConnReuse bool
}

// newConnectDialer creates a dialer to issue CONNECT requests and tunnel traffic via HTTP/S proxy.
Expand Down
44 changes: 22 additions & 22 deletions example/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,52 +35,52 @@ type TlsApiResponse struct {
HTTPVersion string `json:"http_version"`
Method string `json:"method"`
TLS struct {
Ciphers []string `json:"ciphers"`
Extensions []struct {
TLSVersionRecord string `json:"tls_version_record"`
TLSVersionNegotiated string `json:"tls_version_negotiated"`
Ja3 string `json:"ja3"`
Ja3Hash string `json:"ja3_hash"`
ClientRandom string `json:"client_random"`
SessionID string `json:"session_id"`
Ciphers []string `json:"ciphers"`
Extensions []struct {
EllipticCurvesPointFormats interface{} `json:"elliptic_curves_point_formats,omitempty"`
Name string `json:"name"`
ServerName string `json:"server_name,omitempty"`
Data string `json:"data,omitempty"`
PskKeyExchangeMode string `json:"PSK_Key_Exchange_Mode,omitempty"`
SupportedGroups []string `json:"supported_groups,omitempty"`
EllipticCurvesPointFormats interface{} `json:"elliptic_curves_point_formats,omitempty"`
Protocols []string `json:"protocols,omitempty"`
StatusRequest struct {
SignatureAlgorithms []string `json:"signature_algorithms,omitempty"`
SharedKeys []struct {
TLSGrease0X7A7A string `json:"TLS_GREASE (0x7a7a),omitempty"`
X2551929 string `json:"X25519 (29),omitempty"`
} `json:"shared_keys,omitempty"`
Versions []string `json:"versions,omitempty"`
Algorithms []string `json:"algorithms,omitempty"`
StatusRequest struct {
CertificateStatusType string `json:"certificate_status_type"`
ResponderIDListLength int `json:"responder_id_list_length"`
RequestExtensionsLength int `json:"request_extensions_length"`
} `json:"status_request,omitempty"`
SignatureAlgorithms []string `json:"signature_algorithms,omitempty"`
SharedKeys []struct {
TLSGrease0X7A7A string `json:"TLS_GREASE (0x7a7a),omitempty"`
X2551929 string `json:"X25519 (29),omitempty"`
} `json:"shared_keys,omitempty"`
PskKeyExchangeMode string `json:"PSK_Key_Exchange_Mode,omitempty"`
Versions []string `json:"versions,omitempty"`
Algorithms []string `json:"algorithms,omitempty"`
PaddingDataLength int `json:"padding_data_length,omitempty"`
PaddingDataLength int `json:"padding_data_length,omitempty"`
} `json:"extensions"`
TLSVersionRecord string `json:"tls_version_record"`
TLSVersionNegotiated string `json:"tls_version_negotiated"`
Ja3 string `json:"ja3"`
Ja3Hash string `json:"ja3_hash"`
ClientRandom string `json:"client_random"`
SessionID string `json:"session_id"`
} `json:"tls"`
HTTP2 struct {
AkamaiFingerprint string `json:"akamai_fingerprint"`
AkamaiFingerprintHash string `json:"akamai_fingerprint_hash"`
SentFrames []struct {
FrameType string `json:"frame_type"`
Length int `json:"length"`
Settings []string `json:"settings,omitempty"`
Increment int `json:"increment,omitempty"`
StreamID int `json:"stream_id,omitempty"`
Headers []string `json:"headers,omitempty"`
Flags []string `json:"flags,omitempty"`
Priority struct {
Weight int `json:"weight"`
DependsOn int `json:"depends_on"`
Exclusive int `json:"exclusive"`
} `json:"priority,omitempty"`
Length int `json:"length"`
Increment int `json:"increment,omitempty"`
StreamID int `json:"stream_id,omitempty"`
} `json:"sent_frames"`
} `json:"http2"`
HTTP1 struct {
Expand Down
2 changes: 1 addition & 1 deletion jar.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@ import (
type CookieJarOption func(config *cookieJarConfig)

type cookieJarConfig struct {
logger Logger
skipExisting bool
debug bool
allowEmptyCookies bool
logger Logger
}

func WithSkipExisting() CookieJarOption {
Expand Down
4 changes: 2 additions & 2 deletions profiles/profiles.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,12 @@ var MappedTLSClients = map[string]ClientProfile{

type ClientProfile struct {
clientHelloId tls.ClientHelloID
connectionFlow uint32
headerPriority *http2.PriorityParam
settings map[http2.SettingID]uint32
priorities []http2.Priority
pseudoHeaderOrder []string
settings map[http2.SettingID]uint32
settingsOrder []http2.SettingID
connectionFlow uint32
}

func NewClientProfile(clientHelloId tls.ClientHelloID, settings map[http2.SettingID]uint32, settingsOrder []http2.SettingID, pseudoHeaderOrder []string, connectionFlow uint32, priorities []http2.Priority, headerPriority *http2.PriorityParam) ClientProfile {
Expand Down
Loading

0 comments on commit 38ad3c6

Please sign in to comment.