Enable additional ccxt params and http headers

.gitignore

@@ -7,3 +7,4 @@ vendor/
 build/
 bin/
 .idea
+kelp.prefs

.travis.yml

@@ -7,7 +7,8 @@ services:
 
 go:
 - "1.10.x"
-- "1.11"
+- "1.11.x"
+- "1.12.x"
 
 before_install:
 - curl https://glide.sh/get | sh

README.md

@@ -119,6 +119,8 @@ The `trade` command has three required parameters which are:
 - **strategy**: the strategy you want to run (_sell_, _buysell_, _balanced_, _mirror_, _delete_).
 - **stratConf**: full path to the _.cfg_ file specific to your chosen strategy, [sample files here](examples/configs/trader/).
 
+Kelp sets the `X-App-Name` and `X-App-Version` headers on requests made to Horizon. These headers help us track overall Kelp usage, so that we can learn about general usage patterns and adapt Kelp to be more useful in the future. These can be turned off using the `--no-headers` flag. See `kelp trade --help` for more information.
+
 Here's an example of how to start the trading bot with the _buysell_ strategy:
 
 `kelp trade --botConf ./path/trader.cfg --strategy buysell --stratConf ./path/buysell.cfg`

api/exchange.go

@@ -14,6 +14,18 @@ type ExchangeAPIKey struct {
 	Secret string
 }
 
+// ExchangeParam specifies an additional parameter to be sent when initializing the exchange
+type ExchangeParam struct {
+	Param string
+	Value string
+}
+
+// ExchangeHeader specifies additional HTTP headers
+type ExchangeHeader struct {
+	Header string
+	Value  string
+}
+
 // Account allows you to access key account functions
 type Account interface {
 	GetAccountBalances(assetList []interface{}) (map[interface{}]model.Number, error)
@@ -73,6 +85,8 @@ type FillTrackable interface {
 type Constrainable interface {
 	// return nil if the constraint does not exist for the exchange
 	GetOrderConstraints(pair *model.TradingPair) *model.OrderConstraints
+
+	OverrideOrderConstraints(pair *model.TradingPair, override *model.OrderConstraintsOverride)
 }
 
 // OrderbookFetcher extracts out the method that should go into ExchangeShim for now

cmd/terminate.go

@@ -38,8 +38,10 @@ func init() {
 
 		// --- start initialization of objects ----
 		client := &horizon.Client{
-			URL:  configFile.HorizonURL,
-			HTTP: http.DefaultClient,
+			URL:        configFile.HorizonURL,
+			HTTP:       http.DefaultClient,
+			AppName:    "kelp",
+			AppVersion: version,
 		}
 		sdex := plugins.MakeSDEX(
 			client,

cmd/trade.go

@@ -14,20 +14,24 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/stellar/go/build"
 	"github.com/stellar/go/clients/horizon"
+	horizonclient "github.com/stellar/go/exp/clients/horizon"
 	"github.com/stellar/go/support/config"
 	"github.com/stellar/kelp/api"
 	"github.com/stellar/kelp/model"
 	"github.com/stellar/kelp/plugins"
 	"github.com/stellar/kelp/support/logger"
 	"github.com/stellar/kelp/support/monitoring"
 	"github.com/stellar/kelp/support/networking"
+	"github.com/stellar/kelp/support/prefs"
 	"github.com/stellar/kelp/support/utils"
 	"github.com/stellar/kelp/trader"
 )
 
 const tradeExamples = `  kelp trade --botConf ./path/trader.cfg --strategy buysell --stratConf ./path/buysell.cfg
   kelp trade --botConf ./path/trader.cfg --strategy buysell --stratConf ./path/buysell.cfg --sim`
 
+const prefsFilename = "kelp.prefs"
+
 var tradeCmd = &cobra.Command{
 	Use:     "trade",
 	Short:   "Trades against the Stellar universal marketplace using the specified strategy",
@@ -64,6 +68,7 @@ type inputs struct {
 	simMode                       *bool
 	logPrefix                     *string
 	fixedIterations               *uint64
+	noHeaders                     *bool
 }
 
 func validateCliParams(l logger.Logger, options inputs) {
@@ -88,8 +93,19 @@ func validateBotConfig(l logger.Logger, botConfig trader.BotConfig) {
 		logger.Fatal(l, fmt.Errorf("The `FEE` object needs to exist in the trader config file when trading on SDEX"))
 	}
 
-	if !botConfig.IsTradingSdex() && botConfig.MinCentralizedBaseVolume == 0.0 {
-		logger.Fatal(l, fmt.Errorf("need to specify non-zero MIN_CENTRALIZED_BASE_VOLUME config param in trader config file when not trading on SDEX"))
+	if !botConfig.IsTradingSdex() && botConfig.CentralizedMinBaseVolumeOverride != nil && *botConfig.CentralizedMinBaseVolumeOverride <= 0.0 {
+		logger.Fatal(l, fmt.Errorf("need to specify positive CENTRALIZED_MIN_BASE_VOLUME_OVERRIDE config param in trader config file when not trading on SDEX"))
+	}
+	if !botConfig.IsTradingSdex() && botConfig.CentralizedMinQuoteVolumeOverride != nil && *botConfig.CentralizedMinQuoteVolumeOverride <= 0.0 {
+		logger.Fatal(l, fmt.Errorf("need to specify positive CENTRALIZED_MIN_QUOTE_VOLUME_OVERRIDE config param in trader config file when not trading on SDEX"))
+	}
+	validatePrecisionConfig(l, botConfig.IsTradingSdex(), botConfig.CentralizedVolumePrecisionOverride, "CENTRALIZED_VOLUME_PRECISION_OVERRIDE")
+	validatePrecisionConfig(l, botConfig.IsTradingSdex(), botConfig.CentralizedPricePrecisionOverride, "CENTRALIZED_PRICE_PRECISION_OVERRIDE")
+}
+
+func validatePrecisionConfig(l logger.Logger, isTradingSdex bool, precisionField *int8, name string) {
+	if !isTradingSdex && precisionField != nil && *precisionField < 0 {
+		logger.Fatal(l, fmt.Errorf("need to specify non-negative %s config param in trader config file when not trading on SDEX", name))
 	}
 }
 
@@ -105,6 +121,7 @@ func init() {
 	options.simMode = tradeCmd.Flags().Bool("sim", false, "simulate the bot's actions without placing any trades")
 	options.logPrefix = tradeCmd.Flags().StringP("log", "l", "", "log to a file (and stdout) with this prefix for the filename")
 	options.fixedIterations = tradeCmd.Flags().Uint64("iter", 0, "only run the bot for the first N iterations (defaults value 0 runs unboundedly)")
+	options.noHeaders = tradeCmd.Flags().Bool("no-headers", false, "do not set X-App-Name and X-App-Version headers on requests to horizon")
 
 	requiredFlag("botConf")
 	requiredFlag("strategy")
@@ -125,16 +142,16 @@ func makeStartupMessage(options inputs) string {
 	return startupMessage
 }
 
-func makeFeeFn(l logger.Logger, botConfig trader.BotConfig) plugins.OpFeeStroops {
+func makeFeeFn(l logger.Logger, botConfig trader.BotConfig, newClient *horizonclient.Client) plugins.OpFeeStroops {
 	if !botConfig.IsTradingSdex() {
 		return plugins.SdexFixedFeeFn(0)
 	}
 
 	feeFn, e := plugins.SdexFeeFnFromStats(
-		botConfig.HorizonURL,
 		botConfig.Fee.CapacityTrigger,
 		botConfig.Fee.Percentile,
 		botConfig.Fee.MaxOpFeeStroops,
+		newClient,
 	)
 	if e != nil {
 		logger.Fatal(l, fmt.Errorf("could not set up feeFn correctly: %s", e))
@@ -171,6 +188,7 @@ func makeExchangeShimSdex(
 	botConfig trader.BotConfig,
 	options inputs,
 	client *horizon.Client,
+	newClient *horizonclient.Client,
 	ieif *plugins.IEIF,
 	network build.Network,
 	threadTracker *multithreading.ThreadTracker,
@@ -187,21 +205,64 @@ func makeExchangeShimSdex(
 			})
 		}
 
+		exchangeParams := []api.ExchangeParam{}
+		for _, param := range botConfig.ExchangeParams {
+			exchangeParams = append(exchangeParams, api.ExchangeParam{
+				Param: param.Param,
+				Value: param.Value,
+			})
+		}
+
+		exchangeHeaders := []api.ExchangeHeader{}
+		for _, header := range botConfig.ExchangeHeaders {
+			exchangeHeaders = append(exchangeHeaders, api.ExchangeHeader{
+				Header: header.Header,
+				Value:  header.Value,
+			})
+		}
+
 		var exchangeAPI api.Exchange
-		exchangeAPI, e = plugins.MakeTradingExchange(botConfig.TradingExchange, exchangeAPIKeys, *options.simMode)
+		exchangeAPI, e = plugins.MakeTradingExchange(botConfig.TradingExchange, exchangeAPIKeys, exchangeParams, exchangeHeaders, *options.simMode)
 		if e != nil {
 			logger.Fatal(l, fmt.Errorf("unable to make trading exchange: %s", e))
 			return nil, nil
 		}
 
 		exchangeShim = plugins.MakeBatchedExchange(exchangeAPI, *options.simMode, botConfig.AssetBase(), botConfig.AssetQuote(), botConfig.TradingAccount())
+
+		// update precision overrides
+		exchangeShim.OverrideOrderConstraints(tradingPair, model.MakeOrderConstraintsOverride(
+			botConfig.CentralizedPricePrecisionOverride,
+			botConfig.CentralizedVolumePrecisionOverride,
+			nil,
+			nil,
+		))
+		if botConfig.CentralizedMinBaseVolumeOverride != nil {
+			// use updated precision overrides to convert the minCentralizedBaseVolume to a model.Number
+			exchangeShim.OverrideOrderConstraints(tradingPair, model.MakeOrderConstraintsOverride(
+				nil,
+				nil,
+				model.NumberFromFloat(*botConfig.CentralizedMinBaseVolumeOverride, exchangeShim.GetOrderConstraints(tradingPair).VolumePrecision),
+				nil,
+			))
+		}
+		if botConfig.CentralizedMinQuoteVolumeOverride != nil {
+			// use updated precision overrides to convert the minCentralizedQuoteVolume to a model.Number
+			minQuoteVolume := model.NumberFromFloat(*botConfig.CentralizedMinQuoteVolumeOverride, exchangeShim.GetOrderConstraints(tradingPair).VolumePrecision)
+			exchangeShim.OverrideOrderConstraints(tradingPair, model.MakeOrderConstraintsOverride(
+				nil,
+				nil,
+				nil,
+				&minQuoteVolume,
+			))
+		}
 	}
 
 	sdexAssetMap := map[model.Asset]horizon.Asset{
 		tradingPair.Base:  botConfig.AssetBase(),
 		tradingPair.Quote: botConfig.AssetQuote(),
 	}
-	feeFn := makeFeeFn(l, botConfig)
+	feeFn := makeFeeFn(l, botConfig, newClient)
 	sdex := plugins.MakeSDEX(
 		client,
 		ieif,
@@ -287,17 +348,12 @@ func makeBot(
 	if e != nil {
 		l.Infof("Unable to set up monitoring for alert type '%s' with the given API key\n", botConfig.AlertType)
 	}
-	minCentralizedBaseVolume := &botConfig.MinCentralizedBaseVolume
-	if botConfig.IsTradingSdex() {
-		minCentralizedBaseVolume = nil
-	}
 	bot := trader.MakeBot(
 		client,
 		ieif,
 		botConfig.AssetBase(),
 		botConfig.AssetQuote(),
 		tradingPair,
-		minCentralizedBaseVolume,
 		botConfig.TradingAccount(),
 		sdex,
 		exchangeShim,
@@ -331,13 +387,37 @@ func runTradeCmd(options inputs) {
 		URL:  botConfig.HorizonURL,
 		HTTP: http.DefaultClient,
 	}
+	newClient := &horizonclient.Client{
+		// TODO horizonclient.Client has a bug in it where it does not use "/" to separate the horizonURL from the fee_stats endpoint
+		HorizonURL: botConfig.HorizonURL + "/",
+		HTTP:       http.DefaultClient,
+	}
+	if !*options.noHeaders {
+		client.AppName = "kelp"
+		client.AppVersion = version
+		newClient.AppName = "kelp"
+		newClient.AppVersion = version
+
+		p := prefs.Make(prefsFilename)
+		if p.FirstTime() {
+			log.Printf("Kelp sets the `X-App-Name` and `X-App-Version` headers on requests made to Horizon. These headers help us track overall Kelp usage, so that we can learn about general usage patterns and adapt Kelp to be more useful in the future. These can be turned off using the `--no-headers` flag. See `kelp trade --help` for more information.\n")
+			e := p.SetNotFirstTime()
+			if e != nil {
+				l.Info("")
+				l.Errorf("unable to create preferences file: %s", e)
+				// we can still proceed with this error
+			}
+		}
+	}
+
 	ieif := plugins.MakeIEIF(botConfig.IsTradingSdex())
 	network := utils.ParseNetwork(botConfig.HorizonURL)
 	exchangeShim, sdex := makeExchangeShimSdex(
 		l,
 		botConfig,
 		options,
 		client,
+		newClient,
 		ieif,
 		network,
 		threadTracker,
@@ -403,34 +483,36 @@ func runTradeCmd(options inputs) {
 }
 
 func startMonitoringServer(l logger.Logger, botConfig trader.BotConfig) error {
-	serverConfig := &networking.Config{
-		GoogleClientID:     botConfig.GoogleClientID,
-		GoogleClientSecret: botConfig.GoogleClientSecret,
-		PermittedEmails:    map[string]bool{},
-	}
-	// Load acceptable Google emails into the map
-	for _, email := range strings.Split(botConfig.AcceptableEmails, ",") {
-		serverConfig.PermittedEmails[email] = true
-	}
-
 	healthMetrics, e := monitoring.MakeMetricsRecorder(map[string]interface{}{"success": true})
 	if e != nil {
-		return fmt.Errorf("unable to make metrics recorder for the health endpoint: %s", e)
+		return fmt.Errorf("unable to make metrics recorder for the /health endpoint: %s", e)
 	}
-
 	healthEndpoint, e := monitoring.MakeMetricsEndpoint("/health", healthMetrics, networking.NoAuth)
 	if e != nil {
 		return fmt.Errorf("unable to make /health endpoint: %s", e)
 	}
+
 	kelpMetrics, e := monitoring.MakeMetricsRecorder(nil)
 	if e != nil {
 		return fmt.Errorf("unable to make metrics recorder for the /metrics endpoint: %s", e)
 	}
-
-	metricsEndpoint, e := monitoring.MakeMetricsEndpoint("/metrics", kelpMetrics, networking.GoogleAuth)
+	metricsAuth := networking.NoAuth
+	if botConfig.GoogleClientID != "" || botConfig.GoogleClientSecret != "" {
+		metricsAuth = networking.GoogleAuth
+	}
+	metricsEndpoint, e := monitoring.MakeMetricsEndpoint("/metrics", kelpMetrics, metricsAuth)
 	if e != nil {
 		return fmt.Errorf("unable to make /metrics endpoint: %s", e)
 	}
+
+	serverConfig := &networking.Config{
+		GoogleClientID:     botConfig.GoogleClientID,
+		GoogleClientSecret: botConfig.GoogleClientSecret,
+		PermittedEmails:    map[string]bool{},
+	}
+	for _, email := range strings.Split(botConfig.AcceptableEmails, ",") {
+		serverConfig.PermittedEmails[email] = true
+	}
 	server, e := networking.MakeServer(serverConfig, []networking.Endpoint{healthEndpoint, metricsEndpoint})
 	if e != nil {
 		return fmt.Errorf("unable to initialize the metrics server: %s", e)

examples/configs/trader/sample_mirror.cfg

@@ -36,15 +36,31 @@ VOLUME_DIVIDE_BY=500.0
 # in this example the spread is 0.5%
 PER_LEVEL_SPREAD=0.005
 
-# minimum volume of base units needed to place an order on the backing exchange
 # minimum values for Kraken: https://support.kraken.com/hc/en-us/articles/205893708-What-is-the-minimum-order-size-volume-
 # minimum order value for Binance: https://support.binance.com/hc/en-us/articles/115000594711-Trading-Rule
-MIN_BASE_VOLUME=30.0
+# (optional) number of decimal units to be used for price, which is specified in units of the quote asset, needed to place an order on the backing exchange
+#PRICE_PRECISION_OVERRIDE=6
+# (optional) number of decimal units to be used for volume, which is specified in units of the base asset, needed to place an order on the backing exchange
+#VOLUME_PRECISION_OVERRIDE=1
+# (optional) minimum volume of base units needed to place an order on the backing exchange
+#MIN_BASE_VOLUME_OVERRIDE=30.0
+# (optional) minimum volume of quote units needed to place an order on the backing exchange
+#MIN_QUOTE_VOLUME_OVERRIDE=30.0
 
 # set to true if you want the bot to offset your trades onto the backing exchange to realize the per_level_spread against each trade
 # requires you to specify the EXCHANGE_API_KEYS below
 #OFFSET_TRADES=true
 # you can use multiple API keys to overcome rate limit concerns
 #[[EXCHANGE_API_KEYS]]
 #KEY=""
-#SECRET=""
+#SECRET=""
+
+# if your exchange requires additional parameters, list them here with the the necessary values (only ccxt supported currently)
+#[[EXCHANGE_PARAMS]]
+#PARAM=""
+#VALUE=""
+
+# if your exchange requires additional headers, list them here with the the necessary values (only ccxt supported currently)
+#[[EXCHANGE_HEADERS]]
+#HEADER=""
+#VALUE=""