Enable centralized trading2 (#118), closes #102

* 1 - GetAccountBalances API and impls.

* 2 - new interface types in exchange, submittable exchang and Balance

* 3 - typo in ccxtExchange.go

* 4 - merge in changes first attempt

* 5 - remove redundant check of fixedIterations

* 6 - add check for valid submitMode for non-sdex exchange

* 7 - use balance and offers hack methods in trader/trader.go

* 8 - incorporate changes into trade.go and trader.go

* 9 - explicilty disable fill tracking for non-sdex exchanges

* 10 - cursor support to ccxt.go#FetchMyTrades

* 11 - log file name, don't log native by default

* 12 - errorSuffix in batchedExchange log line on submit result

* 13 !! - fix krakenExchange tradeHistory cursor logic

* 14 - sdex tradingOnSdex

* 15 - more %.7f -> %.8f changes

* 16 - wrap error when cannot load open orders for kraken

* 17 - API fixes for MakeIEIF

* 18 - correct name of /health endpoint

* 19 - use 7 for SdexPrecision

* 20 - remove redundant log lines in factory.go

* 21 - add check for ccxt trading needing exactly 1 API key

* 22 - update sample_trader.cfg

* 23 - rename SubmittableExchange to ExchangeShim

* 24 - make FEE config field optional when not trading on SDEX

* 25 - use exchange precision when converting from manageOffer2Order

* 26 - use large precision for intermediate conversion in manageOffer2Order; log orderConstraints

* 27 - include minQuote volume in cost, displayed when logging orderConstraints

api/exchange.go

@@ -3,6 +3,8 @@ package api
 import (
 	"fmt"
 
+	"github.com/stellar/go/build"
+	"github.com/stellar/go/clients/horizon"
 	"github.com/stellar/kelp/model"
 )
 
@@ -14,7 +16,7 @@ type ExchangeAPIKey struct {
 
 // Account allows you to access key account functions
 type Account interface {
-	GetAccountBalances(assetList []model.Asset) (map[model.Asset]model.Number, error)
+	GetAccountBalances(assetList []interface{}) (map[interface{}]model.Number, error)
 }
 
 // Ticker encapsulates all the data for a given Trading Pair
@@ -191,3 +193,17 @@ type Exchange interface {
 	DepositAPI
 	WithdrawAPI
 }
+
+// Balance repesents various aspects of an asset's balance
+type Balance struct {
+	Balance float64
+	Trust   float64
+	Reserve float64
+}
+
+// ExchangeShim is the interface we use as a generic API for all crypto exchanges
+type ExchangeShim interface {
+	SubmitOps(ops []build.TransactionMutator, asyncCallback func(hash string, e error)) error
+	GetBalanceHack(asset horizon.Asset) (*Balance, error)
+	LoadOffersHack() ([]horizon.Offer, error)
+}

cmd/terminate.go

@@ -43,6 +43,8 @@ func init() {
 		}
 		sdex := plugins.MakeSDEX(
 			client,
+			plugins.MakeIEIF(true), // used true for now since it's only ever been tested on SDEX and uses SDEX's data for now
+			nil,
 			configFile.SourceSecretSeed,
 			configFile.TradingSecretSeed,
 			*configFile.SourceAccount,

cmd/trade.go

@@ -89,9 +89,9 @@ func init() {
 		}
 	}
 
-	validateBotConfig := func(l logger.Logger, botConfig trader.BotConfig) {
-		if botConfig.Fee == nil {
-			logger.Fatal(l, fmt.Errorf("The `FEE` object needs to exist in the trader config file"))
+	validateBotConfig := func(l logger.Logger, botConfig trader.BotConfig, isTradingSdex bool) {
+		if isTradingSdex && botConfig.Fee == nil {
+			logger.Fatal(l, fmt.Errorf("The `FEE` object needs to exist in the trader config file when trading on SDEX"))
 		}
 	}
 
@@ -105,9 +105,14 @@ func init() {
 			logger.Fatal(l, e)
 		}
 
+		isTradingSdex := botConfig.TradingExchange == "" || botConfig.TradingExchange == "sdex"
+
 		if *logPrefix != "" {
 			t := time.Now().Format("20060102T150405MST")
 			fileName := fmt.Sprintf("%s_%s_%s_%s_%s_%s.log", *logPrefix, botConfig.AssetCodeA, botConfig.IssuerA, botConfig.AssetCodeB, botConfig.IssuerB, t)
+			if !isTradingSdex {
+				fileName = fmt.Sprintf("%s_%s_%s_%s.log", *logPrefix, botConfig.AssetCodeA, botConfig.AssetCodeB, t)
+			}
 			e = setLogFile(fileName)
 			if e != nil {
 				logger.Fatal(l, e)
@@ -130,7 +135,7 @@ func init() {
 
 		// only log botConfig file here so it can be included in the log file
 		utils.LogConfig(botConfig)
-		validateBotConfig(l, botConfig)
+		validateBotConfig(l, botConfig, isTradingSdex)
 		l.Infof("Trading %s:%s for %s:%s\n", botConfig.AssetCodeA, botConfig.IssuerA, botConfig.AssetCodeB, botConfig.IssuerB)
 
 		client := &horizon.Client{
@@ -144,29 +149,57 @@ func init() {
 		}
 		// --- start initialization of objects ----
 		threadTracker := multithreading.MakeThreadTracker()
+		ieif := plugins.MakeIEIF(isTradingSdex)
 
 		assetBase := botConfig.AssetBase()
 		assetQuote := botConfig.AssetQuote()
 		tradingPair := &model.TradingPair{
 			Base:  model.Asset(utils.Asset2CodeString(assetBase)),
 			Quote: model.Asset(utils.Asset2CodeString(assetQuote)),
 		}
-
 		sdexAssetMap := map[model.Asset]horizon.Asset{
 			tradingPair.Base:  assetBase,
 			tradingPair.Quote: assetQuote,
 		}
-		feeFn, e := plugins.SdexFeeFnFromStats(
-			botConfig.HorizonURL,
-			botConfig.Fee.CapacityTrigger,
-			botConfig.Fee.Percentile,
-			botConfig.Fee.MaxOpFeeStroops,
-		)
-		if e != nil {
-			logger.Fatal(l, fmt.Errorf("could not set up feeFn correctly: %s", e))
+		var feeFn plugins.OpFeeStroops
+		if isTradingSdex {
+			feeFn, e = plugins.SdexFeeFnFromStats(
+				botConfig.HorizonURL,
+				botConfig.Fee.CapacityTrigger,
+				botConfig.Fee.Percentile,
+				botConfig.Fee.MaxOpFeeStroops,
+			)
+			if e != nil {
+				logger.Fatal(l, fmt.Errorf("could not set up feeFn correctly: %s", e))
+			}
+		} else {
+			feeFn = plugins.SdexFixedFeeFn(0)
+		}
+
+		var exchangeShim api.ExchangeShim
+		if !isTradingSdex {
+			exchangeAPIKeys := []api.ExchangeAPIKey{}
+			for _, apiKey := range botConfig.ExchangeAPIKeys {
+				exchangeAPIKeys = append(exchangeAPIKeys, api.ExchangeAPIKey{
+					Key:    apiKey.Key,
+					Secret: apiKey.Secret,
+				})
+			}
+
+			var exchangeAPI api.Exchange
+			exchangeAPI, e = plugins.MakeTradingExchange(botConfig.TradingExchange, exchangeAPIKeys, *simMode)
+			if e != nil {
+				logger.Fatal(l, fmt.Errorf("unable to make trading exchange: %s", e))
+				return
+			}
+
+			exchangeShim = plugins.MakeBatchedExchange(exchangeAPI, *simMode, botConfig.AssetBase(), botConfig.AssetQuote(), botConfig.TradingAccount())
 		}
+
 		sdex := plugins.MakeSDEX(
 			client,
+			ieif,
+			exchangeShim,
 			botConfig.SourceSecretSeed,
 			botConfig.TradingSecretSeed,
 			botConfig.SourceAccount(),
@@ -180,43 +213,60 @@ func init() {
 			sdexAssetMap,
 			feeFn,
 		)
+		if isTradingSdex {
+			exchangeShim = sdex
+		}
 
 		// setting the temp hack variables for the sdex price feeds
-		e = plugins.SetPrivateSdexHack(client, utils.ParseNetwork(botConfig.HorizonURL))
+		e = plugins.SetPrivateSdexHack(client, plugins.MakeIEIF(true), utils.ParseNetwork(botConfig.HorizonURL))
 		if e != nil {
 			l.Info("")
 			l.Errorf("%s", e)
 			// we want to delete all the offers and exit here since there is something wrong with our setup
-			deleteAllOffersAndExit(l, botConfig, client, sdex)
+			deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
 		}
 
 		dataKey := model.MakeSortedBotKey(assetBase, assetQuote)
-		strat, e := plugins.MakeStrategy(sdex, tradingPair, &assetBase, &assetQuote, *strategy, *stratConfigPath, *simMode)
+		strat, e := plugins.MakeStrategy(sdex, ieif, tradingPair, &assetBase, &assetQuote, *strategy, *stratConfigPath, *simMode)
 		if e != nil {
 			l.Info("")
 			l.Errorf("%s", e)
 			// we want to delete all the offers and exit here since there is something wrong with our setup
-			deleteAllOffersAndExit(l, botConfig, client, sdex)
+			deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
 		}
 
 		submitMode, e := api.ParseSubmitMode(botConfig.SubmitMode)
 		if e != nil {
 			log.Println()
 			log.Println(e)
 			// we want to delete all the offers and exit here since there is something wrong with our setup
-			deleteAllOffersAndExit(l, botConfig, client, sdex)
+			deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
+		}
+		if !isTradingSdex && submitMode != api.SubmitModeBoth {
+			log.Println()
+			log.Println("cannot run on a non-SDEX exchange with SUBMIT_MODE set to something other than \"both\"")
+			// we want to delete all the offers and exit here since there is something wrong with our setup
+			deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
+		}
+		if !isTradingSdex && botConfig.FillTrackerSleepMillis != 0 {
+			log.Println()
+			log.Println("cannot run on a non-SDEX exchange with FILL_TRACKER_SLEEP_MILLIS set to something other than 0")
+			// we want to delete all the offers and exit here since there is something wrong with our setup
+			deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
 		}
 		timeController := plugins.MakeIntervalTimeController(
 			time.Duration(botConfig.TickIntervalSeconds)*time.Second,
 			botConfig.MaxTickDelayMillis,
 		)
 		bot := trader.MakeBot(
 			client,
+			ieif,
 			botConfig.AssetBase(),
 			botConfig.AssetQuote(),
 			tradingPair,
 			botConfig.TradingAccount(),
 			sdex,
+			exchangeShim,
 			strat,
 			timeController,
 			botConfig.DeleteCyclesThreshold,
@@ -228,9 +278,13 @@ func init() {
 		)
 		// --- end initialization of objects ---
 
-		l.Info("validating trustlines...")
-		validateTrustlines(l, client, &botConfig)
-		l.Info("trustlines valid")
+		if isTradingSdex {
+			log.Printf("validating trustlines...\n")
+			validateTrustlines(l, client, &botConfig)
+			l.Info("trustlines valid")
+		} else {
+			l.Info("no need to validate trustlines because we're not using SDEX as the trading exchange")
+		}
 
 		// --- start initialization of services ---
 		if botConfig.MonitoringPort != 0 {
@@ -243,7 +297,7 @@ func init() {
 					// we want to delete all the offers and exit here because we don't want the bot to run if monitoring isn't working
 					// if monitoring is desired but not working properly, we want the bot to be shut down and guarantee that there
 					// aren't outstanding offers.
-					deleteAllOffersAndExit(l, botConfig, client, sdex)
+					deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
 				}
 			}()
 		}
@@ -253,7 +307,7 @@ func init() {
 			l.Info("")
 			l.Info("problem encountered while instantiating the fill tracker:")
 			l.Errorf("%s", e)
-			deleteAllOffersAndExit(l, botConfig, client, sdex)
+			deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
 		}
 		if botConfig.FillTrackerSleepMillis != 0 {
 			fillTracker := plugins.MakeFillTracker(tradingPair, threadTracker, sdex, botConfig.FillTrackerSleepMillis)
@@ -273,14 +327,14 @@ func init() {
 					l.Info("problem encountered while running the fill tracker:")
 					l.Errorf("%s", e)
 					// we want to delete all the offers and exit here because we don't want the bot to run if fill tracking isn't working
-					deleteAllOffersAndExit(l, botConfig, client, sdex)
+					deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
 				}
 			}()
 		} else if strategyFillHandlers != nil && len(strategyFillHandlers) > 0 {
 			l.Info("")
 			l.Error("error: strategy has FillHandlers but fill tracking was disabled (set FILL_TRACKER_SLEEP_MILLIS to a non-zero value)")
 			// we want to delete all the offers and exit here because we don't want the bot to run if fill tracking isn't working
-			deleteAllOffersAndExit(l, botConfig, client, sdex)
+			deleteAllOffersAndExit(l, botConfig, client, sdex, exchangeShim)
 		}
 		// --- end initialization of services ---
 
@@ -353,7 +407,7 @@ func validateTrustlines(l logger.Logger, client *horizon.Client, botConfig *trad
 	}
 }
 
-func deleteAllOffersAndExit(l logger.Logger, botConfig trader.BotConfig, client *horizon.Client, sdex *plugins.SDEX) {
+func deleteAllOffersAndExit(l logger.Logger, botConfig trader.BotConfig, client *horizon.Client, sdex *plugins.SDEX, exchangeShim api.ExchangeShim) {
 	l.Info("")
 	l.Info("deleting all offers and then exiting...")
 
@@ -369,7 +423,7 @@ func deleteAllOffersAndExit(l logger.Logger, botConfig trader.BotConfig, client
 	l.Infof("created %d operations to delete offers\n", len(dOps))
 
 	if len(dOps) > 0 {
-		e := sdex.SubmitOps(dOps, func(hash string, e error) {
+		e := exchangeShim.SubmitOps(dOps, func(hash string, e error) {
 			if e != nil {
 				logger.Fatal(l, e)
 				return

examples/configs/trader/sample_trader.cfg

@@ -21,6 +21,7 @@ TICK_INTERVAL_SECONDS=300
 MAX_TICK_DELAY_MILLIS=0
 
 # the mode to use when submitting - maker_only, both (default)
+# when trading on a non-SDEX exchange the only supported mode is "both"
 SUBMIT_MODE="both"
 
 # how many continuous errors in each update cycle can the bot accept before it will delete all offers to protect its exposure.
@@ -33,6 +34,7 @@ SUBMIT_MODE="both"
 # example: use 2 if you want to tolerate 2 continuous update cycle with errors, i.e. three continuous update cycles with errors will delete all offers.
 DELETE_CYCLES_THRESHOLD=0
 # how many milliseconds to sleep before checking for fills again, a value of 0 disables fill tracking
+# fill tracking is not supported when trading on a non-SDEX exchange (i.e. set it to 0)
 FILL_TRACKER_SLEEP_MILLIS=0
 # the url for your horizon instance. If this url contains the string "test" then the bot assumes it is using the test network.
 HORIZON_URL="https://horizon-testnet.stellar.org"
@@ -73,3 +75,14 @@ MAX_OP_FEE_STROOPS=5000
 # a comma-separated list of emails (Google accounts) that are allowed access to monitoring endpoints that require
 # Google authentication.
 #ACCEPTABLE_GOOGLE_EMAILS=""
+
+# uncomment lines below to use kraken. Can use "sdex" or leave out to trade on the Stellar Decentralized Exchange.
+# can alternatively use any of the ccxt-exchanges marked as "Supports Trading" (run `kelp exchanges` for full list)
+#TRADING_EXCHANGE="kraken"
+# you can use multiple API keys to overcome rate limit concerns for kraken
+#[[EXCHANGE_API_KEYS]]
+#KEY=""
+#SECRET=""
+#[[EXCHANGE_API_KEYS]]
+#KEY=""
+#SECRET=""

model/assets.go

@@ -4,6 +4,9 @@ import (
 	"errors"
 	"fmt"
 	"log"
+
+	"github.com/stellar/go/clients/horizon"
+	"github.com/stellar/kelp/support/utils"
 )
 
 // Asset is typed and enlists the allowed assets that are understood by the bot
@@ -168,3 +171,11 @@ var KrakenAssetConverterOpenOrders = makeAssetConverter(map[Asset]string{
 	BTC: "XBT",
 	USD: "USD",
 })
+
+// FromHorizonAsset is a factory method
+func FromHorizonAsset(hAsset horizon.Asset) Asset {
+	if hAsset.Type == utils.Native {
+		return XLM
+	}
+	return Asset(hAsset.Code)
+}