new priceFeedFilter, refactor filterOps, remove dedupeRedundantUpdate…

…sFilter, closes #329 (#335)

* 1 - priceFeedFilter.go

* 2 - better function and var names for priceFeedFilter

* 3 - add filterPriceFeed to filterFactory.go

* 4 - add sample price feed filter in sample_trader.cfg

* 5 - add a log line per op in the priceFeedFilter

* 6 - refactor filterFn to remove keep return var

returning a nil newOp means we don't want to keep it, and returning a non-nil newOp means we want to keep it. Not keeping is handled differently for ops that have existing offers vs. those that do not

* f 6 - need to copy original offer

* 7 - remove dedupeRedundantUpdatesFilter

* 8 - filterOps to fetch original offer if exists for all ops

* 9 - simplify runInnerFilterFn by figuring out filterCounterToIncrement in caller

* 10 - rename !isNewOpNil -> keep

* 11 - consolidate drop case for volumeFilter and makerModeFilter

* 12 - return newOpToPrepend before the newOpToAppend value

cmd/trade.go

@@ -387,9 +387,6 @@ func makeBot(
 		}
 		submitFilters = append(submitFilters, filter)
 	}
-	// always add the dedupeRedundantUpdatesFilter as the second-last filter
-	dedupeRedundantUpdatesFilter := plugins.MakeFilterDedupeRedundantUpdates(assetBase, assetQuote)
-	submitFilters = append(submitFilters, dedupeRedundantUpdatesFilter)
 	// exchange constraints filter is last so we catch any modifications made by previous filters. this ensures that the exchange is
 	// less likely to reject our updates
 	submitFilters = append(submitFilters,

examples/configs/trader/sample_trader.cfg

@@ -120,6 +120,17 @@ HORIZON_URL="https://horizon-testnet.stellar.org"
 #
 #    # limit offers based on a maximum price requirement
 #    "price/max/1.00",
+#
+#    # limit offers based on a reference price from any price feed.
+#    # this "priceFeed" filter uses the format: priceFeed/<comparisonMode>/<feedDataType>/<feedURL>
+#    # and only allows prices "outside" the reference price.
+#    # i.e. gte/gt for sell offers or lte/lt for buy offers based on the comparisonModes options defined:
+#    #     - "outside-exclude" - keeps offers that are great than the reference price for sell offers and keeps offers that
+#    #                           are less than the reference price for buy offers.
+#    #     - "outside-include" - keeps offers that are great than or equal to the reference price for sell offers and keeps
+#    #                           offers that are less than or equal to the reference price for buy offers.
+#    # Note: the feedURL specified at the end of this filter may have its own "/" delimiters which is ok.
+#    "priceFeed/outside-exclude/exchange/kraken/XXLM/ZUSD/mid",
 #]
 
 # specify parameters for how we compute the operation fee from the /fee_stats endpoint

plugins/filterFactory.go

@@ -11,8 +11,9 @@ import (
 )
 
 var filterMap = map[string]func(f *FilterFactory, configInput string) (SubmitFilter, error){
-	"volume": filterVolume,
-	"price":  filterPrice,
+	"volume":    filterVolume,
+	"price":     filterPrice,
+	"priceFeed": filterPriceFeed,
 }
 
 // FilterFactory is a struct that handles creating all the filters
@@ -100,3 +101,26 @@ func filterPrice(f *FilterFactory, configInput string) (SubmitFilter, error) {
 	}
 	return nil, fmt.Errorf("invalid price filter type in second argument (%s)", configInput)
 }
+
+func filterPriceFeed(f *FilterFactory, configInput string) (SubmitFilter, error) {
+	// parts[0] = "priceFeed", parts[1] = comparisonMode, parts[2] = feedDataType, parts[3] = feedURL which can have more "/" chars
+	parts := strings.Split(configInput, "/")
+	if len(parts) < 4 {
+		return nil, fmt.Errorf("\"priceFeed\" filter needs at least 4 parts separated by the '/' delimiter (priceFeed/<comparisonMode>/<feedDataType>/<feedURL>) but we received %s", configInput)
+	}
+
+	cmString := parts[1]
+	feedType := parts[2]
+	feedURL := strings.Join(parts[3:len(parts)], "/")
+	pf, e := MakePriceFeed(feedType, feedURL)
+	if e != nil {
+		return nil, fmt.Errorf("could not make price feed for config input string '%s': %s", configInput, e)
+	}
+
+	filter, e := MakeFilterPriceFeed(f.BaseAsset, f.QuoteAsset, cmString, pf)
+	if e != nil {
+		return nil, fmt.Errorf("could not make price feed filter for config input string '%s': %s", configInput, e)
+	}
+
+	return filter, nil
+}

plugins/makerModeFilter.go

@@ -42,14 +42,14 @@ func (f *makerModeFilter) Apply(ops []txnbuild.Operation, sellingOffers []hProto
 		return nil, fmt.Errorf("could not get assets: %s", e)
 	}
 
-	innerFn := func(op *txnbuild.ManageSellOffer) (*txnbuild.ManageSellOffer, bool, error) {
+	innerFn := func(op *txnbuild.ManageSellOffer) (*txnbuild.ManageSellOffer, error) {
 		topBidPrice, e := f.topOrderPriceExcludingTrader(ob.Bids(), buyingOffers, false)
 		if e != nil {
-			return nil, false, fmt.Errorf("could not get topOrderPriceExcludingTrader for bids: %s", e)
+			return nil, fmt.Errorf("could not get topOrderPriceExcludingTrader for bids: %s", e)
 		}
 		topAskPrice, e := f.topOrderPriceExcludingTrader(ob.Asks(), sellingOffers, true)
 		if e != nil {
-			return nil, false, fmt.Errorf("could not get topOrderPriceExcludingTrader for asks: %s", e)
+			return nil, fmt.Errorf("could not get topOrderPriceExcludingTrader for asks: %s", e)
 		}
 
 		return f.transformOfferMakerMode(baseAsset, quoteAsset, topBidPrice, topAskPrice, op)
@@ -131,20 +131,20 @@ func (f *makerModeFilter) transformOfferMakerMode(
 	topBidPrice *model.Number,
 	topAskPrice *model.Number,
 	op *txnbuild.ManageSellOffer,
-) (*txnbuild.ManageSellOffer, bool, error) {
+) (*txnbuild.ManageSellOffer, error) {
 	// delete operations should never be dropped
 	if op.Amount == "0" {
-		return op, true, nil
+		return op, nil
 	}
 
 	isSell, e := utils.IsSelling(baseAsset, quoteAsset, op.Selling, op.Buying)
 	if e != nil {
-		return nil, false, fmt.Errorf("error when running the isSelling check: %s", e)
+		return nil, fmt.Errorf("error when running the isSelling check: %s", e)
 	}
 
 	sellPrice, e := strconv.ParseFloat(op.Price, 64)
 	if e != nil {
-		return nil, false, fmt.Errorf("could not convert price (%s) to float: %s", op.Price, e)
+		return nil, fmt.Errorf("could not convert price (%s) to float: %s", op.Price, e)
 	}
 
 	var keep bool
@@ -167,18 +167,9 @@ func (f *makerModeFilter) transformOfferMakerMode(
 	}
 
 	if keep {
-		return op, true, nil
+		return op, nil
 	}
 
-	// figure out how to convert the offer to a dropped state
-	if op.OfferID == 0 {
-		// new offers can be dropped
-		return nil, false, nil
-	} else if op.Amount != "0" {
-		// modify offers should be converted to delete offers
-		opCopy := *op
-		opCopy.Amount = "0"
-		return &opCopy, false, nil
-	}
-	return nil, keep, fmt.Errorf("unable to transform manageOffer operation: offerID=%d, amount=%s, price=%.7f", op.OfferID, op.Amount, sellPrice)
+	// we don't want to keep it so return the dropped command
+	return nil, nil
 }

plugins/maxPriceFilter.go

@@ -54,24 +54,24 @@ func (f *maxPriceFilter) Apply(ops []txnbuild.Operation, sellingOffers []hProtoc
 	return ops, nil
 }
 
-func (f *maxPriceFilter) maxPriceFilterFn(op *txnbuild.ManageSellOffer) (*txnbuild.ManageSellOffer, bool, error) {
+func (f *maxPriceFilter) maxPriceFilterFn(op *txnbuild.ManageSellOffer) (*txnbuild.ManageSellOffer, error) {
 	isSell, e := utils.IsSelling(f.baseAsset, f.quoteAsset, op.Selling, op.Buying)
 	if e != nil {
-		return nil, false, fmt.Errorf("error when running the isSelling check: %s", e)
+		return nil, fmt.Errorf("error when running the isSelling check: %s", e)
 	}
 
 	sellPrice, e := strconv.ParseFloat(op.Price, 64)
 	if e != nil {
-		return nil, false, fmt.Errorf("could not convert price (%s) to float: %s", op.Price, e)
+		return nil, fmt.Errorf("could not convert price (%s) to float: %s", op.Price, e)
 	}
 
 	if isSell {
 		if sellPrice > *f.config.MaxPrice {
-			return nil, false, nil
+			return nil, nil
 		}
-		return op, true, nil
+		return op, nil
 	}
 
 	// TODO for buy side
-	return op, true, nil
+	return op, nil
 }

plugins/minPriceFilter.go

@@ -54,24 +54,24 @@ func (f *minPriceFilter) Apply(ops []txnbuild.Operation, sellingOffers []hProtoc
 	return ops, nil
 }
 
-func (f *minPriceFilter) minPriceFilterFn(op *txnbuild.ManageSellOffer) (*txnbuild.ManageSellOffer, bool, error) {
+func (f *minPriceFilter) minPriceFilterFn(op *txnbuild.ManageSellOffer) (*txnbuild.ManageSellOffer, error) {
 	isSell, e := utils.IsSelling(f.baseAsset, f.quoteAsset, op.Selling, op.Buying)
 	if e != nil {
-		return nil, false, fmt.Errorf("error when running the isSelling check: %s", e)
+		return nil, fmt.Errorf("error when running the isSelling check: %s", e)
 	}
 
 	sellPrice, e := strconv.ParseFloat(op.Price, 64)
 	if e != nil {
-		return nil, false, fmt.Errorf("could not convert price (%s) to float: %s", op.Price, e)
+		return nil, fmt.Errorf("could not convert price (%s) to float: %s", op.Price, e)
 	}
 
 	if isSell {
 		if sellPrice < *f.config.MinPrice {
-			return nil, false, nil
+			return nil, nil
 		}
-		return op, true, nil
+		return op, nil
 	}
 
 	// TODO for buy side
-	return op, true, nil
+	return op, nil
 }

plugins/priceFeedFilter.go

@@ -0,0 +1,101 @@
+package plugins
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+
+	hProtocol "github.com/stellar/go/protocols/horizon"
+	"github.com/stellar/go/txnbuild"
+	"github.com/stellar/kelp/api"
+	"github.com/stellar/kelp/support/utils"
+)
+
+type comparisonMode int8
+
+const (
+	comparisonModeOutsideExclude comparisonMode = iota // gt for sell, lt for buy
+	comparisonModeOutsideInclude                       // gte for sell, lte for buy
+)
+
+func (c comparisonMode) keepSellOp(threshold float64, price float64) bool {
+	if c == comparisonModeOutsideExclude {
+		return price > threshold
+	} else if c == comparisonModeOutsideInclude {
+		return price >= threshold
+	}
+	panic("unidentified comparisonMode")
+}
+
+// TODO implement passesBuy() where we use < and <= operators
+
+type priceFeedFilter struct {
+	name       string
+	baseAsset  hProtocol.Asset
+	quoteAsset hProtocol.Asset
+	pf         api.PriceFeed
+	cm         comparisonMode
+}
+
+// MakeFilterPriceFeed makes a submit filter that limits orders placed based on the value of the price feed
+func MakeFilterPriceFeed(baseAsset hProtocol.Asset, quoteAsset hProtocol.Asset, comparisonModeString string, pf api.PriceFeed) (SubmitFilter, error) {
+	var cm comparisonMode
+	if comparisonModeString == "outside-exclude" {
+		cm = comparisonModeOutsideExclude
+	} else if comparisonModeString == "outside-include" {
+		cm = comparisonModeOutsideInclude
+	} else {
+		return nil, fmt.Errorf("invalid comparisonMode ('%s') used for priceFeedFilter", comparisonModeString)
+	}
+
+	return &priceFeedFilter{
+		name:       "priceFeedFilter",
+		baseAsset:  baseAsset,
+		quoteAsset: quoteAsset,
+		cm:         cm,
+		pf:         pf,
+	}, nil
+}
+
+var _ SubmitFilter = &priceFeedFilter{}
+
+func (f *priceFeedFilter) Apply(ops []txnbuild.Operation, sellingOffers []hProtocol.Offer, buyingOffers []hProtocol.Offer) ([]txnbuild.Operation, error) {
+	ops, e := filterOps(f.name, f.baseAsset, f.quoteAsset, sellingOffers, buyingOffers, ops, f.priceFeedFilterFn)
+	if e != nil {
+		return nil, fmt.Errorf("could not apply filter: %s", e)
+	}
+	return ops, nil
+}
+
+func (f *priceFeedFilter) priceFeedFilterFn(op *txnbuild.ManageSellOffer) (*txnbuild.ManageSellOffer, error) {
+	isSell, e := utils.IsSelling(f.baseAsset, f.quoteAsset, op.Selling, op.Buying)
+	if e != nil {
+		return nil, fmt.Errorf("error when running the isSelling check: %s", e)
+	}
+
+	sellPrice, e := strconv.ParseFloat(op.Price, 64)
+	if e != nil {
+		return nil, fmt.Errorf("could not convert price (%s) to float: %s", op.Price, e)
+	}
+
+	thresholdFeedPrice, e := f.pf.GetPrice()
+	if e != nil {
+		return nil, fmt.Errorf("could not get price from priceFeed: %s", e)
+	}
+
+	var opRet *txnbuild.ManageSellOffer
+	// for the sell side we keep only those ops that meet the comparison mode using the value from the price feed as the threshold
+	if isSell {
+		if f.cm.keepSellOp(thresholdFeedPrice, sellPrice) {
+			opRet = op
+		} else {
+			opRet = nil
+		}
+	} else {
+		// for the buy side we keep only those ops that meet the comparison mode using the value from the price feed as the threshold
+		// TODO for buy side (after considering whether sellPrice needs to be inverted or not)
+		opRet = op
+	}
+	log.Printf("priceFeedFilter: isSell=%v, sellPrice=%.10f, thresholdFeedPrice=%.10f, keep=%v", isSell, sellPrice, thresholdFeedPrice, opRet != nil)
+	return opRet, nil
+}