Merge with master + improvements

.github/workflows/configs-json-lint.yml

@@ -0,0 +1,24 @@
+name: configs-json-lint
+on: [push, pull_request]
+
+jobs:
+  lint:
+    name: configs JSON lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check configs JSON format
+        run: |
+          files=("config_example.json" "testdata/configtest.json")
+          for file in "${files[@]}"; do
+            processed_file="${file%.*}_processed.${file##*.}"
+            jq '.exchanges |= sort_by(.name)' --indent 1 $file > $processed_file
+            if ! diff $file $processed_file; then
+              echo "jq differences found in $file! Please run 'make lint_configs'"
+              exit 1
+            else
+              rm $processed_file
+              echo "No differences found in $file 🌞"
+            fi
+          done

.github/workflows/linter.yml

@@ -9,8 +9,7 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version: '1.22.x'
-          cache: false
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v4
+        uses: golangci/golangci-lint-action@v5
         with:
           version: v1.56.1

.github/workflows/proto-lint.yml

@@ -21,7 +21,7 @@ jobs:
           go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
           go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
 
-      - uses: bufbuild/buf-setup-action@v1.30.1
+      - uses: bufbuild/buf-setup-action@v1.31.0
 
       - name: buf generate
         working-directory: ./gctrpc

Makefile

@@ -61,3 +61,20 @@ endif
 target/sqlboiler.json:
 	mkdir -p $(@D)
 	go run ./cmd/gen_sqlboiler_config/main.go $(CONFIG_FLAG) -outdir $(@D)
+
+.PHONY: lint_configs
+lint_configs: check-jq
+	@$(call sort-json,config_example.json)
+	@$(call sort-json,testdata/configtest.json)
+
+define sort-json
+	@printf "Processing $(1)... "
+	@jq '.exchanges |= sort_by(.name)' --indent 1 $(1) > $(1).temp && \
+		(mv $(1).temp $(1) && printf "OK\n") || \
+		(rm $(1).temp; printf "FAILED\n"; exit 1)
+endef
+
+.PHONY: check-jq
+check-jq:
+	@printf "Checking if jq is installed... "
+	@command -v jq >/dev/null 2>&1 && { printf "OK\n"; } || { printf "FAILED. Please install jq to proceed.\n"; exit 1; }

cmd/exchange_wrapper_standards/exchange_wrapper_standards_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/thrasher-corp/gocryptotrader/exchanges/fundingrate"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/futures"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/kline"
+	"github.com/thrasher-corp/gocryptotrader/exchanges/lbank"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/margin"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/order"
 	"github.com/thrasher-corp/gocryptotrader/exchanges/request"
@@ -65,12 +66,6 @@ func TestAllExchangeWrappers(t *testing.T) {
 				ctx, cancelFn = context.WithTimeout(context.Background(), 0)
 				cancelFn()
 			}
-
-			// blocked exchanges are not able to be tested at any level
-			if common.StringDataContains(blockedExchanges, name) {
-				t.Skipf("skipping blocked exchange %v", name)
-			}
-
 			exch, assetPairs := setupExchange(ctx, t, name, cfg)
 			executeExchangeWrapperTests(ctx, t, exch, assetPairs)
 		})
@@ -90,20 +85,34 @@ func setupExchange(ctx context.Context, t *testing.T, name string, cfg *config.C
 		t.Fatalf("Cannot setup %v GetExchangeConfig %v", name, err)
 	}
 	exch.SetDefaults()
-	exchCfg.API.AuthenticatedSupport = true
-	exchCfg.API.Credentials = getExchangeCredentials(name)
-
 	err = exch.Setup(exchCfg)
 	if err != nil {
 		t.Fatalf("Cannot setup %v exchange Setup %v", name, err)
 	}
-
 	err = exch.UpdateTradablePairs(ctx, true)
 	if err != nil && !errors.Is(err, context.DeadlineExceeded) {
 		t.Fatalf("Cannot setup %v UpdateTradablePairs %v", name, err)
 	}
 	b := exch.GetBase()
-
+	// Strange ordering since Setup must be run before UpdateTradablePairs, but Coinbase will fail if invalid
+	// credentials have been set, so we must set them after UpdateTradablePairs
+	creds := getExchangeCredentials(name)
+	b.API.AuthenticatedSupport = true
+	b.API.SetClientID(creds.ClientID)
+	b.API.SetKey(creds.Key)
+	b.API.SetSecret(creds.Secret)
+	b.API.SetPEMKey(creds.PEMKey)
+	b.API.SetSubAccount(creds.Subaccount)
+	// Lbank usually runs this during setup, but if keys aren't set then, it will fail, so we have to manually
+	// recreate that here
+	switch theExch := exch.(type) {
+	case *lbank.Lbank:
+		err = theExch.LoadPrivKey(ctx)
+		if err != nil {
+			t.Fatalf("Cannot setup %v LoadPrivKey %v", name, err)
+		}
+		b.API.AuthenticatedSupport = true
+	}
 	assets := b.CurrencyPairs.GetAssetTypes(false)
 	if len(assets) == 0 {
 		t.Fatalf("Cannot setup %v, exchange has no assets", name)
@@ -114,7 +123,6 @@ func setupExchange(ctx context.Context, t *testing.T, name string, cfg *config.C
 			t.Fatalf("Cannot setup %v SetAssetEnabled %v", name, err)
 		}
 	}
-
 	// Add +1 to len to verify that exchanges can handle requests with unset pairs and assets
 	assetPairs := make([]assetPair, 0, len(assets)+1)
 assets:
@@ -156,7 +164,6 @@ assets:
 		})
 	}
 	assetPairs = append(assetPairs, assetPair{})
-
 	return exch, assetPairs
 }
 
@@ -187,21 +194,18 @@ func executeExchangeWrapperTests(ctx context.Context, t *testing.T, exch exchang
 			continue
 		}
 		method := actualExchange.MethodByName(methodName)
-
 		var assetLen int
 		for y := 0; y < method.Type().NumIn(); y++ {
 			input := method.Type().In(y)
-			if input.AssignableTo(assetParam) ||
-				input.AssignableTo(orderSubmitParam) ||
-				input.AssignableTo(orderModifyParam) ||
-				input.AssignableTo(orderCancelParam) ||
-				input.AssignableTo(orderCancelsParam) ||
-				input.AssignableTo(pairKeySliceParam) ||
-				input.AssignableTo(getOrdersRequestParam) ||
-				input.AssignableTo(pairKeySliceParam) {
-				// this allows wrapper functions that support assets types
-				// to be tested with all supported assets
-				assetLen = len(assetParams) - 1
+			for _, t := range []reflect.Type{
+				assetParam, orderSubmitParam, orderModifyParam, orderCancelParam, orderCancelsParam, pairKeySliceParam, getOrdersRequestParam, latestRateRequest,
+			} {
+				if input.AssignableTo(t) {
+					// this allows wrapper functions that support assets types
+					// to be tested with all supported assets
+					assetLen = len(assetParams) - 1
+					break
+				}
 			}
 		}
 		tt := time.Now()
@@ -389,6 +393,7 @@ func generateMethodArg(ctx context.Context, t *testing.T, argGenerator *MethodAr
 			Description:   "1337",
 			Amount:        1,
 			ClientOrderID: "1337",
+			WalletID:      "7331",
 		}
 		if argGenerator.MethodName == "WithdrawCryptocurrencyFunds" {
 			req.Type = withdraw.Crypto
@@ -604,11 +609,6 @@ var blockedCIExchanges = []string{
 	"bybit",   // bybit API is banned from executing within the US where github Actions is ran
 }
 
-// blockedExchanges are exchanges that are not able to be tested in general
-var blockedExchanges = []string{
-	// "coinbasepro", // coinbasepro API requires authentication for almost every endpoint
-}
-
 // unsupportedAssets contains assets that cannot handle
 // normal processing for testing. This is to be used very sparingly
 var unsupportedAssets = []asset.Item{

cmd/gctcli/commands.go

@@ -4562,3 +4562,90 @@ func getMarginRatesHistory(c *cli.Context) error {
 	jsonOutput(result)
 	return nil
 }
+
+var getCurrencyTradeURLCommand = &cli.Command{
+	Name:      "getcurrencytradeurl",
+	Usage:     "returns the trading url of the instrument",
+	ArgsUsage: "<exchange> <asset> <pair>",
+	Action:    getCurrencyTradeURL,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "exchange",
+			Aliases: []string{"e"},
+			Usage:   "the exchange to retrieve margin rates from",
+		},
+		&cli.StringFlag{
+			Name:    "asset",
+			Aliases: []string{"a"},
+			Usage:   "the asset type of the currency pair",
+		},
+		&cli.StringFlag{
+			Name:    "pair",
+			Aliases: []string{"p"},
+			Usage:   "the currency pair",
+		},
+	},
+}
+
+func getCurrencyTradeURL(c *cli.Context) error {
+	if c.NArg() == 0 && c.NumFlags() == 0 {
+		return cli.ShowSubcommandHelp(c)
+	}
+
+	var exchangeName string
+	if c.IsSet("exchange") {
+		exchangeName = c.String("exchange")
+	} else {
+		exchangeName = c.Args().First()
+	}
+
+	var assetType string
+	if c.IsSet("asset") {
+		assetType = c.String("asset")
+	} else {
+		assetType = c.Args().Get(1)
+	}
+
+	if !validAsset(assetType) {
+		return errInvalidAsset
+	}
+
+	var cp string
+	if c.IsSet("pair") {
+		cp = c.String("pair")
+	} else {
+		cp = c.Args().Get(2)
+	}
+
+	if !validPair(cp) {
+		return errInvalidPair
+	}
+	p, err := currency.NewPairDelimiter(cp, pairDelimiter)
+	if err != nil {
+		return err
+	}
+
+	conn, cancel, err := setupClient(c)
+	if err != nil {
+		return err
+	}
+	defer closeConn(conn, cancel)
+
+	client := gctrpc.NewGoCryptoTraderServiceClient(conn)
+	result, err := client.GetCurrencyTradeURL(c.Context,
+		&gctrpc.GetCurrencyTradeURLRequest{
+			Exchange: exchangeName,
+			Asset:    assetType,
+			Pair: &gctrpc.CurrencyPair{
+				Delimiter: p.Delimiter,
+				Base:      p.Base.String(),
+				Quote:     p.Quote.String(),
+			},
+		})
+	if err != nil {
+		return err
+	}
+
+	jsonOutput(result)
+	return nil
+}

cmd/gctcli/main.go

@@ -218,6 +218,7 @@ func main() {
 		technicalAnalysisCommand,
 		getMarginRatesHistoryCommand,
 		orderbookCommand,
+		getCurrencyTradeURLCommand,
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())

common/common.go

@@ -581,6 +581,34 @@ func ExcludeError(err, excl error) error {
 	return err
 }
 
+// ErrorCollector allows collecting a stream of errors from concurrent go routines
+// Users should call e.Wg.Done and send errors to e.C
+type ErrorCollector struct {
+	C  chan error
+	Wg sync.WaitGroup
+}
+
+// CollectErrors returns an ErrorCollector with WaitGroup and Channel buffer set to n
+func CollectErrors(n int) *ErrorCollector {
+	e := &ErrorCollector{
+		C: make(chan error, n),
+	}
+	e.Wg.Add(n)
+	return e
+}
+
+// Collect runs waits for e.Wg to be Done, closes the error channel, and return a error collection
+func (e *ErrorCollector) Collect() (errs error) {
+	e.Wg.Wait()
+	close(e.C)
+	for err := range e.C {
+		if err != nil {
+			errs = AppendError(errs, err)
+		}
+	}
+	return
+}
+
 // StartEndTimeCheck provides some basic checks which occur
 // frequently in the codebase
 func StartEndTimeCheck(start, end time.Time) error {

common/common_test.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"github.com/thrasher-corp/gocryptotrader/common/file"
 )
 
@@ -836,3 +837,22 @@ func TestGenerateRandomString(t *testing.T) {
 		t.Error("GenerateRandomString() unexpected test validation result")
 	}
 }
+
+// TestErrorCollector exercises the error collector
+func TestErrorCollector(t *testing.T) {
+	e := CollectErrors(4)
+	for i := range 4 {
+		go func() {
+			if i%2 == 0 {
+				e.C <- errors.New("Collected error")
+			} else {
+				e.C <- nil
+			}
+			e.Wg.Done()
+		}()
+	}
+	v := e.Collect()
+	errs, ok := v.(*multiError)
+	require.True(t, ok, "Must return a multiError")
+	assert.Len(t, errs.Unwrap(), 2, "Should have 2 errors")
+}

config/config.go

@@ -448,22 +448,9 @@ func (c *Config) CheckPairConfigFormats(exchName string) error {
 			}
 
 			for y := range loadedPairs {
-				if pairFmt.Delimiter != "" && pairFmt.Index != "" {
-					return fmt.Errorf(
-						"exchange %s %s %s cannot have an index and delimiter set at the same time",
-						exchName, pairsType, assetType)
-				}
 				if pairFmt.Delimiter != "" {
 					if !strings.Contains(loadedPairs[y].String(), pairFmt.Delimiter) {
-						return fmt.Errorf(
-							"exchange %s %s %s pairs does not contain delimiter",
-							exchName, pairsType, assetType)
-					}
-				}
-				if pairFmt.Index != "" {
-					if !strings.Contains(loadedPairs[y].String(), pairFmt.Index) {
-						return fmt.Errorf("exchange %s %s %s pairs does not contain an index",
-							exchName, pairsType, assetType)
+						return fmt.Errorf("exchange %s %s %s pairs does not contain delimiter", exchName, pairsType, assetType)
 					}
 				}
 			}