Merge branch 'gateio_rate_limits' into gateio_sub_out_batch

common/math/math.go

@@ -24,6 +24,10 @@ var (
 	errCAGRNoIntervals         = errors.New("cannot calculate CAGR with no intervals")
 	errCAGRZeroOpenValue       = errors.New("cannot calculate CAGR with an open value of 0")
 	errInformationBadLength    = errors.New("benchmark rates length does not match returns rates")
+
+	one        = decimal.NewFromInt(1)
+	two        = decimal.NewFromInt(2)
+	oneHundred = decimal.NewFromInt(100)
 )
 
 // CalculateAmountWithFee returns a calculated fee included amount on fee
@@ -36,16 +40,22 @@ func CalculateFee(amount, fee float64) float64 {
 	return amount * (fee / 100)
 }
 
-// CalculatePercentageGainOrLoss returns the percentage rise over a certain
-// period
-func CalculatePercentageGainOrLoss(priceNow, priceThen float64) float64 {
-	return (priceNow - priceThen) / priceThen * 100
+// PercentageChange returns the percentage change between two numbers, x is reference value.
+func PercentageChange(x, y float64) float64 {
+	return (y - x) / x * 100
+}
+
+// PercentageDifference returns difference between two numbers as a percentage of their average
+func PercentageDifference(x, y float64) float64 {
+	return math.Abs(x-y) / ((x + y) / 2) * 100
 }
 
-// CalculatePercentageDifference returns the percentage of difference between
-// multiple time periods
-func CalculatePercentageDifference(amount, secondAmount float64) float64 {
-	return (amount - secondAmount) / ((amount + secondAmount) / 2) * 100
+// PercentageDifferenceDecimal returns the difference between two decimal values as a percentage of their average
+func PercentageDifferenceDecimal(x, y decimal.Decimal) decimal.Decimal {
+	if x.IsZero() && y.IsZero() {
+		return decimal.Zero
+	}
+	return x.Sub(y).Abs().Div(x.Add(y).Div(two)).Mul(oneHundred)
 }
 
 // CalculateNetProfit returns net profit
@@ -267,7 +277,7 @@ func DecimalCompoundAnnualGrowthRate(openValue, closeValue, intervalsPerYear, nu
 	if pow.IsZero() {
 		return decimal.Zero, ErrPowerDifferenceTooSmall
 	}
-	k := pow.Sub(decimal.NewFromInt(1)).Mul(decimal.NewFromInt(100))
+	k := pow.Sub(one).Mul(oneHundred)
 	return k, nil
 }
 
@@ -317,7 +327,7 @@ func DecimalPopulationStandardDeviation(values []decimal.Decimal) (decimal.Decim
 	diffs := make([]decimal.Decimal, len(values))
 	for x := range values {
 		val := values[x].Sub(valAvg)
-		exp := decimal.NewFromInt(2)
+		exp := two
 		pow := DecimalPow(val, exp)
 		diffs[x] = pow
 	}
@@ -349,11 +359,11 @@ func DecimalSampleStandardDeviation(values []decimal.Decimal) (decimal.Decimal,
 	superMean := make([]decimal.Decimal, len(values))
 	var combined decimal.Decimal
 	for i := range values {
-		pow := values[i].Sub(mean).Pow(decimal.NewFromInt(2))
+		pow := values[i].Sub(mean).Pow(two)
 		superMean[i] = pow
 		combined.Add(pow)
 	}
-	avg := combined.Div(decimal.NewFromInt(int64(len(superMean))).Sub(decimal.NewFromInt(1)))
+	avg := combined.Div(decimal.NewFromInt(int64(len(superMean))).Sub(one))
 	f, exact := avg.Float64()
 	err = nil
 	if !exact {
@@ -370,15 +380,15 @@ func DecimalGeometricMean(values []decimal.Decimal) (decimal.Decimal, error) {
 	if len(values) == 0 {
 		return decimal.Zero, errZeroValue
 	}
-	product := decimal.NewFromInt(1)
+	product := one
 	for i := range values {
 		if values[i].LessThanOrEqual(decimal.Zero) {
 			// cannot use negative or zero values in geometric calculation
 			return decimal.Zero, errGeometricNegative
 		}
 		product = product.Mul(values[i])
 	}
-	exp := decimal.NewFromInt(1).Div(decimal.NewFromInt(int64(len(values))))
+	exp := one.Div(decimal.NewFromInt(int64(len(values))))
 	pow := DecimalPow(product, exp)
 	geometricPower := pow
 	return geometricPower, nil
@@ -413,7 +423,7 @@ func DecimalFinancialGeometricMean(values []decimal.Decimal) (decimal.Decimal, e
 		// as we cannot have negative or zero value geometric numbers
 		// adding a 1 to the percentage movements allows for differentiation between
 		// negative numbers (eg -0.1 translates to 0.9) and positive numbers (eg 0.1 becomes 1.1)
-		modVal := values[i].Add(decimal.NewFromInt(1)).InexactFloat64()
+		modVal := values[i].Add(one).InexactFloat64()
 		product *= modVal
 	}
 	prod := 1 / float64(len(values))
@@ -446,7 +456,7 @@ func DecimalSortinoRatio(movementPerCandle []decimal.Decimal, riskFreeRatePerInt
 	totalNegativeResultsSquared := decimal.Zero
 	for x := range movementPerCandle {
 		if movementPerCandle[x].Sub(riskFreeRatePerInterval).LessThan(decimal.Zero) {
-			totalNegativeResultsSquared = totalNegativeResultsSquared.Add(movementPerCandle[x].Sub(riskFreeRatePerInterval).Pow(decimal.NewFromInt(2)))
+			totalNegativeResultsSquared = totalNegativeResultsSquared.Add(movementPerCandle[x].Sub(riskFreeRatePerInterval).Pow(two))
 		}
 	}
 	if totalNegativeResultsSquared.IsZero() {

common/math/math_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 
 	"github.com/shopspring/decimal"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestCalculateFee(t *testing.T) {
@@ -28,27 +30,49 @@ func TestCalculateAmountWithFee(t *testing.T) {
 	}
 }
 
-func TestCalculatePercentageGainOrLoss(t *testing.T) {
+func TestPercentageChange(t *testing.T) {
 	t.Parallel()
-	originalInput := float64(9300)
-	secondInput := float64(9000)
-	expectedOutput := 3.3333333333333335
-	actualResult := CalculatePercentageGainOrLoss(originalInput, secondInput)
-	if expectedOutput != actualResult {
-		t.Errorf(
-			"Expected '%v'. Actual '%v'.", expectedOutput, actualResult)
+	assert.Equal(t, 3.3333333333333335, PercentageChange(9000, 9300))
+	assert.Equal(t, -3.225806451612903, PercentageChange(9300, 9000))
+	assert.True(t, math.IsNaN(PercentageChange(0, 0)))
+	assert.Equal(t, 0.0, PercentageChange(1, 1))
+	assert.Equal(t, 0.0, PercentageChange(-1, -1))
+	assert.True(t, math.IsInf(PercentageChange(0, 1), 1))
+	assert.Equal(t, -100., PercentageChange(1, 0))
+}
+
+func TestPercentageDifference(t *testing.T) {
+	t.Parallel()
+	require.Equal(t, 196.03960396039605, PercentageDifference(1, 100))
+	require.Equal(t, 196.03960396039605, PercentageDifference(100, 1))
+	require.Equal(t, 0.13605442176870758, PercentageDifference(1.469, 1.471))
+	require.Equal(t, 0.13605442176870758, PercentageDifference(1.471, 1.469))
+	require.Equal(t, 0.0, PercentageDifference(1.0, 1.0))
+	require.True(t, math.IsNaN(PercentageDifference(0.0, 0.0)))
+}
+
+// 1000000000	         0.2215 ns/op	       0 B/op	       0 allocs/op
+func BenchmarkPercentageDifference(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		PercentageDifference(1.469, 1.471)
 	}
 }
 
-func TestCalculatePercentageDifference(t *testing.T) {
+func TestPercentageDifferenceDecimal(t *testing.T) {
 	t.Parallel()
-	originalInput := float64(10)
-	secondAmount := float64(5)
-	expectedOutput := 66.66666666666666
-	actualResult := CalculatePercentageDifference(originalInput, secondAmount)
-	if expectedOutput != actualResult {
-		t.Errorf(
-			"Expected '%f'. Actual '%f'.", expectedOutput, actualResult)
+	require.Equal(t, "196.03960396039604", PercentageDifferenceDecimal(decimal.NewFromFloat(1), decimal.NewFromFloat(100)).String())
+	require.Equal(t, "196.03960396039604", PercentageDifferenceDecimal(decimal.NewFromFloat(100), decimal.NewFromFloat(1)).String())
+	require.Equal(t, "0.13605442176871", PercentageDifferenceDecimal(decimal.NewFromFloat(1.469), decimal.NewFromFloat(1.471)).String())
+	require.Equal(t, "0.13605442176871", PercentageDifferenceDecimal(decimal.NewFromFloat(1.471), decimal.NewFromFloat(1.469)).String())
+	require.Equal(t, "0", PercentageDifferenceDecimal(decimal.NewFromFloat(1.0), decimal.NewFromFloat(1.0)).String())
+	require.Equal(t, "0", PercentageDifferenceDecimal(decimal.Zero, decimal.Zero).String())
+}
+
+// 1585596	       751.8 ns/op	     792 B/op	      27 allocs/op
+func BenchmarkDecimalPercentageDifference(b *testing.B) {
+	d1, d2 := decimal.NewFromFloat(1.469), decimal.NewFromFloat(1.471)
+	for i := 0; i < b.N; i++ {
+		PercentageDifferenceDecimal(d1, d2)
 	}
 }
 

engine/datahistory_manager.go

@@ -1049,10 +1049,10 @@ func (m *DataHistoryManager) CheckCandleIssue(job *DataHistoryJob, multiplier in
 	}
 	if apiData != dbData {
 		var diff float64
-		if apiData > dbData {
-			diff = gctmath.CalculatePercentageGainOrLoss(apiData, dbData)
+		if apiData < dbData {
+			diff = gctmath.PercentageChange(apiData, dbData)
 		} else {
-			diff = gctmath.CalculatePercentageGainOrLoss(dbData, apiData)
+			diff = gctmath.PercentageChange(dbData, apiData)
 		}
 		if diff > job.IssueTolerancePercentage {
 			issue = fmt.Sprintf("%s api: %v db: %v diff: %v %%", candleField, apiData, dbData, diff)

exchanges/bithumb/bithumb_websocket.go

@@ -5,11 +5,16 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strings"
+	"text/template"
 	"time"
 
+	"github.com/Masterminds/sprig/v3"
 	"github.com/gorilla/websocket"
 	"github.com/thrasher-corp/gocryptotrader/common"
+	"github.com/thrasher-corp/gocryptotrader/currency"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/subscription"
@@ -24,10 +29,15 @@ const (
 )
 
 var (
-	wsDefaultTickTypes = []string{"30M"} // alternatives "1H", "12H", "24H", "MID"
-	location           *time.Location
+	location *time.Location
 )
 
+var defaultSubscriptions = subscription.List{
+	{Enabled: true, Asset: asset.Spot, Channel: subscription.TickerChannel, Interval: kline.ThirtyMin}, // alternatives "1H", "12H", "24H"
+	{Enabled: true, Asset: asset.Spot, Channel: subscription.OrderbookChannel},
+	{Enabled: true, Asset: asset.Spot, Channel: subscription.AllTradesChannel},
+}
+
 // WsConnect initiates a websocket connection
 func (b *Bithumb) WsConnect() error {
 	if !b.Websocket.IsEnabled() || !b.IsEnabled() {
@@ -171,41 +181,19 @@ func (b *Bithumb) wsHandleData(respRaw []byte) error {
 
 // generateSubscriptions generates the default subscription set
 func (b *Bithumb) generateSubscriptions() (subscription.List, error) {
-	var channels = []string{"ticker", "transaction", "orderbookdepth"}
-	var subscriptions subscription.List
-	pairs, err := b.GetEnabledPairs(asset.Spot)
-	if err != nil {
-		return nil, err
-	}
+	return b.Features.Subscriptions.ExpandTemplates(b)
+}
 
-	pFmt, err := b.GetPairFormat(asset.Spot, true)
-	if err != nil {
-		return nil, err
-	}
-	pairs = pairs.Format(pFmt)
-
-	for y := range channels {
-		subscriptions = append(subscriptions, &subscription.Subscription{
-			Channel: channels[y],
-			Pairs:   pairs,
-			Asset:   asset.Spot,
-		})
-	}
-	return subscriptions, nil
+// GetSubscriptionTemplate returns a subscription channel template
+func (b *Bithumb) GetSubscriptionTemplate(_ *subscription.Subscription) (*template.Template, error) {
+	return template.New("master.tmpl").Funcs(sprig.FuncMap()).Funcs(template.FuncMap{"subToReq": subToReq}).Parse(subTplText)
 }
 
 // Subscribe subscribes to a set of channels
-func (b *Bithumb) Subscribe(channelsToSubscribe subscription.List) error {
+func (b *Bithumb) Subscribe(subs subscription.List) error {
 	var errs error
-	for _, s := range channelsToSubscribe {
-		req := &WsSubscribe{
-			Type:    s.Channel,
-			Symbols: s.Pairs,
-		}
-		if s.Channel == "ticker" {
-			req.TickTypes = wsDefaultTickTypes
-		}
-		err := b.Websocket.Conn.SendJSONMessage(context.TODO(), request.Unset, req)
+	for _, s := range subs {
+		err := b.Websocket.Conn.SendJSONMessage(context.TODO(), request.Unset, json.RawMessage(s.QualifiedChannel))
 		if err == nil {
 			err = b.Websocket.AddSuccessfulSubscriptions(b.Websocket.Conn, s)
 		}
@@ -215,3 +203,32 @@ func (b *Bithumb) Subscribe(channelsToSubscribe subscription.List) error {
 	}
 	return errs
 }
+
+// subToReq returns the subscription as a map to populate WsSubscribe
+func subToReq(s *subscription.Subscription, p currency.Pairs) *WsSubscribe {
+	req := &WsSubscribe{
+		Type:    s.Channel,
+		Symbols: common.SortStrings(p),
+	}
+	switch s.Channel {
+	case subscription.TickerChannel:
+		// As-is
+	case subscription.OrderbookChannel:
+		req.Type = "orderbookdepth"
+	case subscription.AllTradesChannel:
+		req.Type = "transaction"
+	default:
+		panic(fmt.Errorf("%w: %s", subscription.ErrNotSupported, s.Channel))
+	}
+	if s.Interval > 0 {
+		req.TickTypes = []string{strings.ToUpper(s.Interval.Short())}
+	}
+	return req
+}
+
+const subTplText = `
+{{ range $asset, $pairs := $.AssetPairs }}
+	{{- subToReq $.S $pairs | mustToJson }}
+	{{- $.AssetSeparator }}
+{{- end }}
+`

exchanges/bithumb/bithumb_websocket_test.go

@@ -4,11 +4,17 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/thrasher-corp/gocryptotrader/currency"
 	exchange "github.com/thrasher-corp/gocryptotrader/exchanges"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/asset"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/stream"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/subscription"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/ticker"
+	testexch "github.com/thrasher-corp/gocryptotrader/internal/testing/exchange"
+	testsubs "github.com/thrasher-corp/gocryptotrader/internal/testing/subscriptions"
 )
 
 var (
@@ -87,13 +93,40 @@ func TestWsHandleData(t *testing.T) {
 	}
 }
 
+func TestSubToReq(t *testing.T) {
+	t.Parallel()
+	p := currency.Pairs{currency.NewPairWithDelimiter("BTC", "KRW", "_"), currency.NewPairWithDelimiter("ETH", "KRW", "_")}
+	r := subToReq(&subscription.Subscription{Channel: subscription.AllTradesChannel}, p)
+	assert.Equal(t, "transaction", r.Type)
+	assert.True(t, p.Equal(r.Symbols))
+	r = subToReq(&subscription.Subscription{Channel: subscription.OrderbookChannel}, p)
+	assert.Equal(t, "orderbookdepth", r.Type)
+	assert.True(t, p.Equal(r.Symbols))
+	r = subToReq(&subscription.Subscription{Channel: subscription.TickerChannel, Interval: kline.OneHour}, p)
+	assert.Equal(t, "ticker", r.Type)
+	assert.True(t, p.Equal(r.Symbols))
+	assert.Equal(t, []string{"1H"}, r.TickTypes)
+	assert.PanicsWithError(t,
+		"subscription channel not supported: myTrades",
+		func() { subToReq(&subscription.Subscription{Channel: subscription.MyTradesChannel}, p) },
+		"should panic on invalid channel",
+	)
+}
+
 func TestGenerateSubscriptions(t *testing.T) {
 	t.Parallel()
-	sub, err := b.generateSubscriptions()
-	if err != nil {
-		t.Fatal(err)
-	}
-	if sub == nil {
-		t.Fatal("unexpected value")
+	b := new(Bithumb)
+	require.NoError(t, testexch.Setup(b), "Test instance Setup must not error")
+	p := currency.Pairs{currency.NewPairWithDelimiter("BTC", "KRW", "_"), currency.NewPairWithDelimiter("ETH", "KRW", "_")}
+	require.NoError(t, b.CurrencyPairs.StorePairs(asset.Spot, p, false))
+	require.NoError(t, b.CurrencyPairs.StorePairs(asset.Spot, p, true))
+	subs, err := b.generateSubscriptions()
+	require.NoError(t, err)
+	exp := subscription.List{
+		{Asset: asset.Spot, Channel: subscription.AllTradesChannel, Pairs: p, QualifiedChannel: `{"type":"transaction","symbols":["BTC_KRW","ETH_KRW"]}`},
+		{Asset: asset.Spot, Channel: subscription.OrderbookChannel, Pairs: p, QualifiedChannel: `{"type":"orderbookdepth","symbols":["BTC_KRW","ETH_KRW"]}`},
+		{Asset: asset.Spot, Channel: subscription.TickerChannel, Pairs: p, Interval: kline.ThirtyMin,
+			QualifiedChannel: `{"type":"ticker","symbols":["BTC_KRW","ETH_KRW"],"tickTypes":["30M"]}`},
 	}
+	testsubs.EqualLists(t, exp, subs)
 }

exchanges/bithumb/bithumb_wrapper.go

@@ -109,6 +109,7 @@ func (b *Bithumb) SetDefaults() {
 				GlobalResultLimit: 1500,
 			},
 		},
+		Subscriptions: defaultSubscriptions.Clone(),
 	}
 	b.Requester, err = request.New(b.Name,
 		common.NewHTTPClientWithTimeout(exchange.DefaultHTTPTimeout),