exp/ticker: Orderbook data support

exp/ticker/cmd/generate.go

@@ -76,7 +76,7 @@ var cmdGenerateAssetData = &cobra.Command{
 		Logger.Infof("Starting asset data generation, outputting to: %s\n", AssetsOutFile)
 		err = ticker.GenerateAssetsFile(&session, Logger, AssetsOutFile)
 		if err != nil {
-			Logger.Fatal("could not generate market data:", err)
+			Logger.Fatal("could not generate asset data:", err)
 		}
 	},
 }

exp/ticker/cmd/ingest.go

@@ -16,6 +16,7 @@ func init() {
 	rootCmd.AddCommand(cmdIngest)
 	cmdIngest.AddCommand(cmdIngestAssets)
 	cmdIngest.AddCommand(cmdIngestTrades)
+	cmdIngest.AddCommand(cmdIngestOrderbooks)
 
 	cmdIngestTrades.Flags().BoolVar(
 		&ShouldStream,
@@ -96,3 +97,26 @@ var cmdIngestTrades = &cobra.Command{
 		}
 	},
 }
+
+var cmdIngestOrderbooks = &cobra.Command{
+	Use:   "orderbooks",
+	Short: "Refreshes the orderbook stats database with new data retrieved from Horizon.",
+	Run: func(cmd *cobra.Command, args []string) {
+		Logger.Info("Refreshing the asset database")
+		dbInfo, err := pq.ParseURL(DatabaseURL)
+		if err != nil {
+			Logger.Fatal("could not parse db-url:", err)
+		}
+
+		session, err := tickerdb.CreateSession("postgres", dbInfo)
+		if err != nil {
+			Logger.Fatal("could not connect to db:", err)
+		}
+		defer session.DB.Close()
+
+		err = ticker.RefreshOrderbookEntries(&session, Client, Logger)
+		if err != nil {
+			Logger.Fatal("could not refresh error database:", err)
+		}
+	},
+}

exp/ticker/docker/conf/crontab.txt

@@ -4,6 +4,9 @@
 # Refresh the database of assets, hourly:
 @hourly /opt/stellar/bin/ticker ingest assets --db-url=postgres://127.0.0.1:5432/ticker > /home/stellar/last-ingest-assets.log 2>&1
 
+# Refresh the database of orderbooks, every 10 minutes:
+*/10 * * * * /opt/stellar/bin/ticker ingest orderbooks --db-url=postgres://127.0.0.1:5432/ticker > /home/stellar/last-ingest-orderbooks.log 2>&1
+
 # Backfill the database of trades (including possible new assets), every 6 hours:
 0 */6 * * * /opt/stellar/bin/ticker ingest trades --db-url=postgres://127.0.0.1:5432/ticker > /home/stellar/last-ingest-trades.log 2>&1
 

exp/ticker/docker/setup

@@ -10,7 +10,7 @@ mkdir -p /opt/stellar/www
 chown -R stellar:stellar /opt/stellar/www
 mkdir -p /opt/stellar/postgresql/data
 
-wget -O ticker.tar.gz https://github.com/accordeiro/ticker-releases/releases/download/v0.4-alpha/ticker.tar.gz
+wget -O ticker.tar.gz https://github.com/accordeiro/ticker-releases/releases/download/v0.5.3-alpha/ticker.tar.gz
 tar -xvzf ticker.tar.gz
 mv ticker /opt/stellar/bin/ticker
 chmod +x /opt/stellar/bin/ticker

exp/ticker/docker/start

@@ -28,6 +28,7 @@ function main() {
 
 	populate_assets
 	populate_trades
+	populate_orderbooks
 	generate_assets_file
 	generate_markets_file
 
@@ -70,6 +71,19 @@ function populate_trades() {
 }
 
 
+function populate_orderbooks() {
+	if [ -f $PGHOME/.orderbooks-populated ]; then
+		echo "ticker: orderbooks already pre-populated"
+		return 0
+	fi
+	echo ""
+	echo "Populating initial orderbook database"
+	echo ""
+	sudo -u stellar $STELLAR_BIN/ticker ingest orderbooks --db-url=$PGURL
+	touch $PGHOME/.orderbooks-populated
+}
+
+
 function generate_assets_file() {
 	if [ -f $STELLAR_HOME/www/assets.json ]; then
 		echo "ticker: assets.json already pre-populated"

exp/ticker/internal/actions_market.go

@@ -59,6 +59,8 @@ func GenerateMarketSummary(s *tickerdb.TickerSession) (ms MarketSummary, err err
 
 func dbMarketToMarketStats(m tickerdb.Market) MarketStats {
 	closeTime := utils.TimeToUnixEpoch(m.LastPriceCloseTime)
+
+	spread, spreadMidPoint := utils.CalcSpread(m.HighestBid, m.LowestAsk)
 	return MarketStats{
 		TradePairName:    m.TradePair,
 		BaseVolume24h:    m.BaseVolume24h,
@@ -77,6 +79,14 @@ func dbMarketToMarketStats(m tickerdb.Market) MarketStats {
 		Change7d:         m.PriceChange7d,
 		Price:            m.LastPrice,
 		Close:            m.LastPrice,
+		NumBids:          m.NumBids,
+		BidVolume:        m.BidVolume,
+		HighestBid:       m.HighestBid,
+		NumAsks:          m.NumAsks,
+		AskVolume:        m.AskVolume,
+		LowestAsk:        m.LowestAsk,
+		Spread:           spread,
+		SpreadMidPoint:   spreadMidPoint,
 		CloseTime:        closeTime,
 	}
 }

exp/ticker/internal/actions_orderbook.go

@@ -0,0 +1,65 @@
+package ticker
+
+import (
+	"time"
+
+	horizonclient "github.com/stellar/go/clients/horizonclient"
+	"github.com/stellar/go/exp/ticker/internal/scraper"
+	"github.com/stellar/go/exp/ticker/internal/tickerdb"
+	"github.com/stellar/go/support/errors"
+	hlog "github.com/stellar/go/support/log"
+)
+
+// RefreshOrderbookEntries updates the orderbook entries for the relevant markets that were active
+// in the past 7-day interval
+func RefreshOrderbookEntries(s *tickerdb.TickerSession, c *horizonclient.Client, l *hlog.Entry) error {
+	sc := scraper.ScraperConfig{
+		Client: c,
+		Logger: l,
+	}
+
+	// Retrieve relevant markets for the past 7 days (168 hours):
+	mkts, err := s.Retrieve7DRelevantMarkets()
+	if err != nil {
+		return errors.Wrap(err, "could not retrieve partial markets")
+	}
+
+	for _, mkt := range mkts {
+		ob, err := sc.FetchOrderbookForAssets(
+			mkt.BaseAssetType,
+			mkt.BaseAssetCode,
+			mkt.BaseAssetIssuer,
+			mkt.CounterAssetType,
+			mkt.CounterAssetCode,
+			mkt.CounterAssetIssuer,
+		)
+		if err != nil {
+			l.Error(errors.Wrap(err, "could not fetch orderbook for assets"))
+			continue
+		}
+
+		dbOS := orderbookStatsToDBOrderbookStats(ob, mkt.BaseAssetID, mkt.CounterAssetID)
+		err = s.InsertOrUpdateOrderbookStats(&dbOS, []string{"base_asset_id", "counter_asset_id"})
+		if err != nil {
+			l.Error(errors.Wrap(err, "could not insert orderbook stats into db"))
+		}
+	}
+
+	return nil
+}
+
+func orderbookStatsToDBOrderbookStats(os scraper.OrderbookStats, bID, cID int32) tickerdb.OrderbookStats {
+	return tickerdb.OrderbookStats{
+		BaseAssetID:    bID,
+		CounterAssetID: cID,
+		NumBids:        os.NumBids,
+		BidVolume:      os.BidVolume,
+		HighestBid:     os.HighestBid,
+		NumAsks:        os.NumAsks,
+		AskVolume:      os.AskVolume,
+		LowestAsk:      os.LowestAsk,
+		Spread:         os.Spread,
+		SpreadMidPoint: os.SpreadMidPoint,
+		UpdatedAt:      time.Now(),
+	}
+}

exp/ticker/internal/gql/main.go

@@ -38,6 +38,7 @@ type asset struct {
 	Countries                   string
 	Status                      string
 	IssuerID                    int32
+	OrderbookStats              orderbookStats
 }
 
 // partialMarket represents the aggregated market data for a
@@ -58,6 +59,20 @@ type partialMarket struct {
 	Close                float64
 	IntervalStart        graphql.Time
 	FirstLedgerCloseTime graphql.Time
+	OrderbookStats       orderbookStats
+}
+
+// orderbookStats represents the orderbook stats for a
+// specific pair of assets (aggregated or not)
+type orderbookStats struct {
+	NumBids        BigInt
+	BidVolume      float64
+	HighestBid     float64
+	NumAsks        BigInt
+	AskVolume      float64
+	LowestAsk      float64
+	Spread         float64
+	SpreadMidPoint float64
 }
 
 type resolver struct {

exp/ticker/internal/gql/resolvers_market.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/graph-gophers/graphql-go"
 	"github.com/stellar/go/exp/ticker/internal/tickerdb"
+	"github.com/stellar/go/exp/ticker/internal/utils"
 )
 
 // Markets resolves the markets() GraphQL query.
@@ -83,6 +84,18 @@ func validateNumHoursAgo(n *int32) (int, error) {
 
 // dbMarketToPartialMarket converts a tickerdb.PartialMarket to a *partialMarket
 func dbMarketToPartialMarket(dbMarket tickerdb.PartialMarket) *partialMarket {
+	spread, spreadMidPoint := utils.CalcSpread(dbMarket.HighestBid, dbMarket.LowestAsk)
+	os := orderbookStats{
+		NumBids:        BigInt(dbMarket.NumBids),
+		BidVolume:      dbMarket.BidVolume,
+		HighestBid:     dbMarket.HighestBid,
+		NumAsks:        BigInt(dbMarket.NumAsks),
+		AskVolume:      dbMarket.AskVolume,
+		LowestAsk:      dbMarket.LowestAsk,
+		Spread:         spread,
+		SpreadMidPoint: spreadMidPoint,
+	}
+
 	return &partialMarket{
 		TradePair:            dbMarket.TradePairName,
 		BaseAssetCode:        dbMarket.BaseAssetCode,
@@ -99,5 +112,6 @@ func dbMarketToPartialMarket(dbMarket tickerdb.PartialMarket) *partialMarket {
 		Close:                dbMarket.Close,
 		IntervalStart:        graphql.Time{Time: dbMarket.IntervalStart},
 		FirstLedgerCloseTime: graphql.Time{Time: dbMarket.FirstLedgerCloseTime},
+		OrderbookStats:       os,
 	}
 }

exp/ticker/internal/gql/static/schema.gql

@@ -74,7 +74,8 @@ type Market {
 	change: Float!
 	close: Float!
 	intervalStart: Time!
-	firstLedgerCloseTime: Time!
+	firstLedgerCloseTime: Time!,
+	orderbookStats: OrderbookStats!
 }
 
 type AggregatedMarket {
@@ -88,7 +89,19 @@ type AggregatedMarket {
 	change: Float!
 	close: Float!
 	intervalStart: Time!
-	firstLedgerCloseTime: Time!
+	firstLedgerCloseTime: Time!,
+	orderbookStats: OrderbookStats!
+}
+
+type OrderbookStats {
+ 	numBids: BigInt!,
+	bidVolume: Float!,
+	highestBid: Float!,
+	numAsks: BigInt!,
+	askVolume: Float!,
+	lowestAsk: Float!,
+	spread: Float!,
+	spreadMidPoint: Float!
 }
 
 type Issuer {

exp/ticker/internal/main.go

@@ -32,6 +32,14 @@ type MarketStats struct {
 	Price            float64 `json:"price"`
 	Close            float64 `json:"close"`
 	CloseTime        int64   `json:"close_time"`
+	NumBids          int     `json:"num_bids"`
+	BidVolume        float64 `json:"bid_volume"`
+	HighestBid       float64 `json:"highest_bid"`
+	NumAsks          int     `json:"num_asks"`
+	AskVolume        float64 `json:"ask_volume"`
+	LowestAsk        float64 `json:"lowest_ask"`
+	Spread           float64 `json:"spread"`
+	SpreadMidPoint   float64 `json:"spread_mid_point"`
 }
 
 // Asset Sumary represents the collection of valid assets.