Skip to content

Commit

Permalink
exchanges/request: abstract and consolidate rate limiting code to req…
Browse files Browse the repository at this point in the history 
…uest package (#1477)

* initial consolidation of rate limiting code to request package to reduce bespoke code implementation

* continued

* finish abstraction

* lint

* exchanges: fix tests

* linter: fix

* poloniex: fix auth rate limit not being set

* ratelimiter: convert from token to weight

* glorious: nits addressed with fire

* linter: rip

* change func name set -> get

* fix test

* derbit: impl

---------

Co-authored-by: Ryan O'Hara-Reid <[email protected]>
  • Loading branch information
@shazbert
shazbert and Ryan O'Hara-Reid authored Jun 3, 2024
1 parent aeb4a87 commit f6a95da
Show file tree
Hide file tree
Showing 64 changed files with 780 additions and 2,577 deletions.
6 changes: 3 additions & 3 deletions cmd/apichecker/apicheck.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1226,11 +1226,11 @@ func sendGetReq(path string, result interface{}) error {
if strings.Contains(path, "github") {
requester, err = request.New("Apichecker",
common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
request.WithLimiter(request.NewBasicRateLimit(time.Hour, 60)))
request.WithLimiter(request.NewBasicRateLimit(time.Hour, 60, 1)))
} else {
requester, err = request.New("Apichecker",
common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
request.WithLimiter(request.NewBasicRateLimit(time.Second, 100)))
request.WithLimiter(request.NewBasicRateLimit(time.Second, 100, 1)))
}
if err != nil {
return err
Expand All @@ -1249,7 +1249,7 @@ func sendGetReq(path string, result interface{}) error {
func sendAuthReq(method, path string, result interface{}) error {
requester, err := request.New("Apichecker",
common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
request.WithLimiter(request.NewBasicRateLimit(time.Second*10, 100)))
request.WithLimiter(request.NewBasicRateLimit(time.Second*10, 100, 1)))
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion currency/coinmarketcap/coinmarketcap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func (c *Coinmarketcap) SetDefaults() {
var err error
c.Requester, err = request.New(c.Name,
common.NewHTTPClientWithTimeout(defaultTimeOut),
request.WithLimiter(request.NewBasicRateLimit(RateInterval, BasicRequestRate)),
request.WithLimiter(request.NewBasicRateLimit(RateInterval, BasicRequestRate, 1)),
)
if err != nil {
log.Errorln(log.Global, err)
Expand Down
2 changes: 1 addition & 1 deletion currency/forexprovider/currencyconverterapi/currencyconverterapi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func (c *CurrencyConverter) Setup(config base.Settings) error {
var err error
c.Requester, err = request.New(c.Name,
common.NewHTTPClientWithTimeout(base.DefaultTimeOut),
request.WithLimiter(request.NewBasicRateLimit(rateInterval, requestRate)))
request.WithLimiter(request.NewBasicRateLimit(rateInterval, requestRate, 1)))
return err
}

Expand Down
2 changes: 1 addition & 1 deletion currency/forexprovider/exchangeratesapi.io/exchangeratesapi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func (e *ExchangeRates) Setup(config base.Settings) error {
var err error
e.Requester, err = request.New(e.Name,
common.NewHTTPClientWithTimeout(base.DefaultTimeOut),
request.WithLimiter(request.NewBasicRateLimit(rateLimitInterval, requestRate)))
request.WithLimiter(request.NewBasicRateLimit(rateLimitInterval, requestRate, 1)))
return err
}

Expand Down
2 changes: 1 addition & 1 deletion exchanges/binance/binance_cfutures.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -722,7 +722,7 @@ func getKlineRateBudget(limit int64) request.EndpointLimit {
rateBudget := cFuturesDefaultRate
switch {
case limit > 0 && limit < 100:
rateBudget = cFuturesKline100Rate
rateBudget = cFuturesDefaultRate
case limit >= 100 && limit < 500:
rateBudget = cFuturesKline500Rate
case limit >= 500 && limit < 1000:
Expand Down
5 changes: 0 additions & 5 deletions exchanges/binance/binance_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -753,11 +753,6 @@ type UserAccountStream struct {
ListenKey string `json:"listenKey"`
}

type wsAccountInfo struct {
Stream string `json:"stream"`
Data WsAccountInfoData `json:"data"`
}

// WsAccountInfoData defines websocket account info data
type WsAccountInfoData struct {
CanDeposit bool `json:"D"`
Expand Down
2 changes: 1 addition & 1 deletion exchanges/binance/binance_wrapper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ func (b *Binance) SetDefaults() {

b.Requester, err = request.New(b.Name,
common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),
request.WithLimiter(SetRateLimit()))
request.WithLimiter(GetRateLimits()))
if err != nil {
log.Errorln(log.ExchangeSys, err)
}
Expand Down
242 changes: 72 additions & 170 deletions exchanges/binance/ratelimit.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
package binance

import (
"context"
"fmt"
"time"

"github.com/thrasher-corp/gocryptotrader/exchanges/request"
"golang.org/x/time/rate"
)

const (
Expand Down Expand Up @@ -77,7 +74,6 @@ const (
cFuturesOrderbook100Rate
cFuturesOrderbook500Rate
cFuturesOrderbook1000Rate
cFuturesKline100Rate
cFuturesKline500Rate
cFuturesKline1000Rate
cFuturesKlineMaxRate
Expand All @@ -96,173 +92,79 @@ const (
uFuturesSetMultiAssetMarginRate
)

// RateLimit implements the request.Limiter interface
type RateLimit struct {
SpotRate *rate.Limiter
SpotOrdersRate *rate.Limiter
UFuturesRate *rate.Limiter
UFuturesOrdersRate *rate.Limiter
CFuturesRate *rate.Limiter
CFuturesOrdersRate *rate.Limiter
}

// Limit executes rate limiting functionality for Binance
func (r *RateLimit) Limit(ctx context.Context, f request.EndpointLimit) error {
var limiter *rate.Limiter
var tokens int
switch f {
case spotDefaultRate:
limiter, tokens = r.SpotRate, 1
case spotOrderbookTickerAllRate,
spotSymbolPriceAllRate:
limiter, tokens = r.SpotRate, 2
case spotHistoricalTradesRate,
spotOrderbookDepth500Rate:
limiter, tokens = r.SpotRate, 5
case spotOrderbookDepth1000Rate,
spotAccountInformationRate,
spotExchangeInfo:
limiter, tokens = r.SpotRate, 10
case spotPriceChangeAllRate:
limiter, tokens = r.SpotRate, 40
case spotOrderbookDepth5000Rate:
limiter, tokens = r.SpotRate, 50
case spotOrderRate:
limiter, tokens = r.SpotOrdersRate, 1
case spotOrderQueryRate:
limiter, tokens = r.SpotOrdersRate, 2
case spotOpenOrdersSpecificRate:
limiter, tokens = r.SpotOrdersRate, 3
case spotAllOrdersRate:
limiter, tokens = r.SpotOrdersRate, 10
case spotOpenOrdersAllRate:
limiter, tokens = r.SpotOrdersRate, 40
case uFuturesDefaultRate,
uFuturesKline100Rate:
limiter, tokens = r.UFuturesRate, 1
case uFuturesOrderbook50Rate,
uFuturesKline500Rate,
uFuturesOrderbookTickerAllRate:
limiter, tokens = r.UFuturesRate, 2
case uFuturesOrderbook100Rate,
uFuturesKline1000Rate,
uFuturesAccountInformationRate:
limiter, tokens = r.UFuturesRate, 5
case uFuturesOrderbook500Rate,
uFuturesKlineMaxRate:
limiter, tokens = r.UFuturesRate, 10
case uFuturesOrderbook1000Rate,
uFuturesHistoricalTradesRate:
limiter, tokens = r.UFuturesRate, 20
case uFuturesTickerPriceHistoryRate:
limiter, tokens = r.UFuturesRate, 40
case uFuturesOrdersDefaultRate:
limiter, tokens = r.UFuturesOrdersRate, 1
case uFuturesBatchOrdersRate,
uFuturesGetAllOrdersRate:
limiter, tokens = r.UFuturesOrdersRate, 5
case uFuturesCountdownCancelRate:
limiter, tokens = r.UFuturesOrdersRate, 10
case uFuturesCurrencyForceOrdersRate,
uFuturesSymbolOrdersRate:
limiter, tokens = r.UFuturesOrdersRate, 20
case uFuturesIncomeHistoryRate:
limiter, tokens = r.UFuturesOrdersRate, 30
case uFuturesPairOrdersRate,
uFuturesGetAllOpenOrdersRate:
limiter, tokens = r.UFuturesOrdersRate, 40
case uFuturesAllForceOrdersRate:
limiter, tokens = r.UFuturesOrdersRate, 50
case cFuturesKline100Rate:
limiter, tokens = r.CFuturesRate, 1
case cFuturesKline500Rate,
cFuturesOrderbookTickerAllRate:
limiter, tokens = r.CFuturesRate, 2
case cFuturesKline1000Rate,
cFuturesAccountInformationRate:
limiter, tokens = r.CFuturesRate, 5
case cFuturesKlineMaxRate,
cFuturesIndexMarkPriceRate:
limiter, tokens = r.CFuturesRate, 10
case cFuturesHistoricalTradesRate,
cFuturesCurrencyForceOrdersRate:
limiter, tokens = r.CFuturesRate, 20
case cFuturesTickerPriceHistoryRate:
limiter, tokens = r.CFuturesRate, 40
case cFuturesAllForceOrdersRate:
limiter, tokens = r.CFuturesRate, 50
case cFuturesOrdersDefaultRate:
limiter, tokens = r.CFuturesOrdersRate, 1
case cFuturesBatchOrdersRate,
cFuturesGetAllOpenOrdersRate:
limiter, tokens = r.CFuturesOrdersRate, 5
case cFuturesCancelAllOrdersRate:
limiter, tokens = r.CFuturesOrdersRate, 10
case cFuturesIncomeHistoryRate,
cFuturesSymbolOrdersRate:
limiter, tokens = r.CFuturesOrdersRate, 20
case cFuturesPairOrdersRate:
limiter, tokens = r.CFuturesOrdersRate, 40
case cFuturesOrderbook50Rate:
limiter, tokens = r.CFuturesRate, 2
case cFuturesOrderbook100Rate:
limiter, tokens = r.CFuturesRate, 5
case cFuturesOrderbook500Rate:
limiter, tokens = r.CFuturesRate, 10
case cFuturesOrderbook1000Rate:
limiter, tokens = r.CFuturesRate, 20
case cFuturesDefaultRate:
limiter, tokens = r.CFuturesRate, 1
case uFuturesMultiAssetMarginRate:
limiter, tokens = r.UFuturesRate, 30
case uFuturesSetMultiAssetMarginRate:
limiter, tokens = r.UFuturesRate, 1
default:
limiter, tokens = r.SpotRate, 1
}

var finalDelay time.Duration
var reserves = make([]*rate.Reservation, tokens)
for i := 0; i < tokens; i++ {
// Consume tokens 1 at a time as this avoids needing burst capacity in the limiter,
// which would otherwise allow the rate limit to be exceeded over short periods
reserves[i] = limiter.Reserve()
finalDelay = reserves[i].Delay()
// GetRateLimits returns the rate limit for the exchange
func GetRateLimits() request.RateLimitDefinitions {
spotDefaultLimiter := request.NewRateLimit(spotInterval, spotRequestRate)
spotOrderLimiter := request.NewRateLimit(spotOrderInterval, spotOrderRequestRate)
usdMarginedFuturesLimiter := request.NewRateLimit(uFuturesInterval, uFuturesRequestRate)
usdMarginedFuturesOrdersLimiter := request.NewRateLimit(uFuturesOrderInterval, uFuturesOrderRequestRate)
coinMarginedFuturesLimiter := request.NewRateLimit(cFuturesInterval, cFuturesRequestRate)
coinMarginedFuturesOrdersLimiter := request.NewRateLimit(cFuturesOrderInterval, cFuturesOrderRequestRate)

return request.RateLimitDefinitions{
spotDefaultRate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 1),
spotOrderbookTickerAllRate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 2),
spotSymbolPriceAllRate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 2),
spotHistoricalTradesRate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 5),
spotOrderbookDepth500Rate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 5),
spotOrderbookDepth1000Rate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 10),
spotAccountInformationRate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 10),
spotExchangeInfo: request.GetRateLimiterWithWeight(spotDefaultLimiter, 10),
spotPriceChangeAllRate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 40),
spotOrderbookDepth5000Rate: request.GetRateLimiterWithWeight(spotDefaultLimiter, 50),
spotOrderRate: request.GetRateLimiterWithWeight(spotOrderLimiter, 1),
spotOrderQueryRate: request.GetRateLimiterWithWeight(spotOrderLimiter, 2),
spotOpenOrdersSpecificRate: request.GetRateLimiterWithWeight(spotOrderLimiter, 3),
spotAllOrdersRate: request.GetRateLimiterWithWeight(spotOrderLimiter, 10),
spotOpenOrdersAllRate: request.GetRateLimiterWithWeight(spotOrderLimiter, 40),
uFuturesDefaultRate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 1),
uFuturesKline100Rate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 1),
uFuturesOrderbook50Rate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 2),
uFuturesKline500Rate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 2),
uFuturesOrderbookTickerAllRate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 2),
uFuturesOrderbook100Rate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 5),
uFuturesKline1000Rate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 5),
uFuturesAccountInformationRate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 5),
uFuturesOrderbook500Rate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 10),
uFuturesKlineMaxRate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 10),
uFuturesOrderbook1000Rate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 20),
uFuturesHistoricalTradesRate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 20),
uFuturesTickerPriceHistoryRate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 40),
uFuturesOrdersDefaultRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 1),
uFuturesBatchOrdersRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 5),
uFuturesGetAllOrdersRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 5),
uFuturesCountdownCancelRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 10),
uFuturesCurrencyForceOrdersRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 20),
uFuturesSymbolOrdersRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 20),
uFuturesIncomeHistoryRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 30),
uFuturesPairOrdersRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 40),
uFuturesGetAllOpenOrdersRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 40),
uFuturesAllForceOrdersRate: request.GetRateLimiterWithWeight(usdMarginedFuturesOrdersLimiter, 50),
cFuturesDefaultRate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 1),
cFuturesKline500Rate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 2),
cFuturesOrderbookTickerAllRate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 2),
cFuturesKline1000Rate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 5),
cFuturesAccountInformationRate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 5),
cFuturesKlineMaxRate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 10),
cFuturesIndexMarkPriceRate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 10),
cFuturesHistoricalTradesRate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 20),
cFuturesCurrencyForceOrdersRate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 20),
cFuturesTickerPriceHistoryRate: request.GetRateLimiterWithWeight(coinMarginedFuturesLimiter, 40),
cFuturesAllForceOrdersRate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 50),
cFuturesOrdersDefaultRate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 1),
cFuturesBatchOrdersRate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 5),
cFuturesGetAllOpenOrdersRate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 5),
cFuturesCancelAllOrdersRate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 10),
cFuturesIncomeHistoryRate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 20),
cFuturesSymbolOrdersRate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 20),
cFuturesPairOrdersRate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 40),
cFuturesOrderbook50Rate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 2),
cFuturesOrderbook100Rate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 5),
cFuturesOrderbook500Rate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 10),
cFuturesOrderbook1000Rate: request.GetRateLimiterWithWeight(coinMarginedFuturesOrdersLimiter, 20),
uFuturesMultiAssetMarginRate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 30),
uFuturesSetMultiAssetMarginRate: request.GetRateLimiterWithWeight(usdMarginedFuturesLimiter, 1),
}

if dl, ok := ctx.Deadline(); ok && dl.Before(time.Now().Add(finalDelay)) {
// Cancel all potential reservations to free up rate limiter if deadline
// is exceeded.
for x := range reserves {
reserves[x].Cancel()
}
return fmt.Errorf("rate limit delay of %s will exceed deadline: %w",
finalDelay,
context.DeadlineExceeded)
}

time.Sleep(finalDelay)
return nil
}

// SetRateLimit returns the rate limit for the exchange
func SetRateLimit() *RateLimit {
return &RateLimit{
SpotRate: request.NewRateLimit(spotInterval, spotRequestRate),
SpotOrdersRate: request.NewRateLimit(spotOrderInterval, spotOrderRequestRate),
UFuturesRate: request.NewRateLimit(uFuturesInterval, uFuturesRequestRate),
UFuturesOrdersRate: request.NewRateLimit(uFuturesOrderInterval, uFuturesOrderRequestRate),
CFuturesRate: request.NewRateLimit(cFuturesInterval, cFuturesRequestRate),
CFuturesOrdersRate: request.NewRateLimit(cFuturesOrderInterval, cFuturesOrderRequestRate),
}
}

func bestPriceLimit(symbol string) request.EndpointLimit {
if symbol == "" {
return spotOrderbookTickerAllRate
}

return spotDefaultRate
}

func openOrdersLimit(symbol string) request.EndpointLimit {
Expand Down
Loading

0 comments on commit f6a95da

Please sign in to comment.