operation_fee_stats endpoint (#586)

services/horizon/internal/actions_operation_fee_stats.go

@@ -0,0 +1,46 @@
+package horizon
+
+import (
+	"fmt"
+
+	"github.com/stellar/go/services/horizon/internal/operationfeestats"
+	"github.com/stellar/go/support/render/hal"
+)
+
+// This file contains the actions:
+//
+// OperationFeeStatsAction: stats representing current state of network fees
+
+// OperationFeeStatsAction renders a few useful statistics that describe the
+// current state of operation fees on the network.
+type OperationFeeStatsAction struct {
+	Action
+	Min         int64
+	Mode        int64
+	LastBaseFee int64
+	LastLedger  int64
+}
+
+// JSON is a method for actions.JSON
+func (action *OperationFeeStatsAction) JSON() {
+	action.Do(
+		action.loadRecords,
+		func() {
+			hal.Render(action.W, map[string]string{
+				"min_accepted_fee":     fmt.Sprint(action.Min),
+				"mode_accepted_fee":    fmt.Sprint(action.Mode),
+				"last_ledger_base_fee": fmt.Sprint(action.LastBaseFee),
+				"last_ledger":          fmt.Sprint(action.LastLedger),
+			})
+		},
+	)
+}
+
+func (action *OperationFeeStatsAction) loadRecords() {
+	cur := operationfeestats.CurrentState()
+
+	action.Min = cur.Min
+	action.Mode = cur.Mode
+	action.LastBaseFee = cur.LastBaseFee
+	action.LastLedger = cur.LastLedger
+}

services/horizon/internal/actions_operation_fee_stats_test.go

@@ -0,0 +1,56 @@
+package horizon
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestOperationFeeTestsActions_Show(t *testing.T) {
+
+	testCases := []struct {
+		scenario    string
+		min         string
+		mode        string
+		lastbasefee string
+	}{
+		// happy path
+		{
+			"operation_fee_stats_1",
+			"100",
+			"100",
+			"100",
+		},
+		// no transactions in last 5 ledgers
+		{
+			"operation_fee_stats_2",
+			"100",
+			"100",
+			"100",
+		},
+		// transactions with varying fees
+		{
+			"operation_fee_stats_3",
+			"200",
+			"400",
+			"100",
+		},
+	}
+
+	for _, kase := range testCases {
+		t.Run("/operation_fee_stats", func(t *testing.T) {
+			ht := StartHTTPTest(t, kase.scenario)
+			defer ht.Finish()
+
+			w := ht.Get("/operation_fee_stats")
+
+			if ht.Assert.Equal(200, w.Code) {
+				var result map[string]string
+				err := json.Unmarshal(w.Body.Bytes(), &result)
+				ht.Require.NoError(err)
+				ht.Assert.Equal(kase.min, result["min_accepted_fee"])
+				ht.Assert.Equal(kase.mode, result["mode_accepted_fee"])
+				ht.Assert.Equal(kase.lastbasefee, result["last_ledger_base_fee"])
+			}
+		})
+	}
+}

services/horizon/internal/app.go

@@ -16,6 +16,7 @@ import (
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/services/horizon/internal/ingest"
 	"github.com/stellar/go/services/horizon/internal/ledger"
+	"github.com/stellar/go/services/horizon/internal/operationfeestats"
 	"github.com/stellar/go/services/horizon/internal/paths"
 	"github.com/stellar/go/services/horizon/internal/reap"
 	"github.com/stellar/go/services/horizon/internal/render/sse"
@@ -184,6 +185,54 @@ Failed:
 
 }
 
+// UpdateOperationFeeStatsState triggers a refresh of several operation fee metrics
+func (a *App) UpdateOperationFeeStatsState() {
+	var err error
+	var next operationfeestats.State
+
+	var latest history.LatestLedger
+	var feeStats history.FeeStats
+
+	cur := operationfeestats.CurrentState()
+
+	err = a.HistoryQ().LatestLedgerBaseFeeAndSequence(&latest)
+	if err != nil {
+		goto Failed
+	}
+
+	// finish early if no new ledgers
+	if cur.LastLedger == int64(latest.Sequence) {
+		return
+	}
+
+	next.LastBaseFee = int64(latest.BaseFee)
+	next.LastLedger = int64(latest.Sequence)
+
+	err = a.HistoryQ().TransactionsForLastXLedgers(latest.Sequence, &feeStats)
+	if err != nil {
+		goto Failed
+	}
+
+	// if no transactions in last X ledgers, return
+	// latest ledger's base fee for all
+	if !feeStats.Mode.Valid && !feeStats.Min.Valid {
+		next.Min = next.LastBaseFee
+		next.Mode = next.LastBaseFee
+	} else {
+		next.Min = feeStats.Min.Int64
+		next.Mode = feeStats.Mode.Int64
+	}
+
+	operationfeestats.SetState(next)
+	return
+
+Failed:
+	log.WithStack(err).
+		WithField("err", err.Error()).
+		Error("failed to load operation fee stats state")
+
+}
+
 // UpdateStellarCoreInfo updates the value of coreVersion and networkPassphrase
 // from the Stellar core API.
 func (a *App) UpdateStellarCoreInfo() {
@@ -235,9 +284,10 @@ func (a *App) DeleteUnretainedHistory() error {
 func (a *App) Tick() {
 	var wg sync.WaitGroup
 	log.Debug("ticking app")
-	// update ledger state and stellar-core info in parallel
-	wg.Add(2)
+	// update ledger state, operation fee state, and stellar-core info in parallel
+	wg.Add(3)
 	go func() { a.UpdateLedgerState(); wg.Done() }()
+	go func() { a.UpdateOperationFeeStatsState(); wg.Done() }()
 	go func() { a.UpdateStellarCoreInfo(); wg.Done() }()
 	wg.Wait()
 

services/horizon/internal/db2/history/main.go

@@ -159,6 +159,20 @@ type EffectsQ struct {
 // `history_effects` table.
 type EffectType int
 
+// FeeStats is a row of data from the min, mode aggregate functions over the
+// `history_ledgers` table.
+type FeeStats struct {
+	Min  null.Int `db:"min"`
+	Mode null.Int `db:"mode"`
+}
+
+// LatestLedger represents a response from the raw LatestLedgerBaseFeeAndSequence
+// query.
+type LatestLedger struct {
+	BaseFee  int32 `db:"base_fee"`
+	Sequence int32 `db:"sequence"`
+}
+
 // Ledger is a row of data from the `history_ledgers` table
 type Ledger struct {
 	TotalOrderID
@@ -300,6 +314,16 @@ func (q *Q) LatestLedger(dest interface{}) error {
 	return q.GetRaw(dest, `SELECT COALESCE(MAX(sequence), 0) FROM history_ledgers`)
 }
 
+// LatestLedgerBaseFeeAndSequence loads the latest known ledger's base fee and
+// sequence number.
+func (q *Q) LatestLedgerBaseFeeAndSequence(dest interface{}) error {
+	return q.GetRaw(dest, `
+		SELECT base_fee, sequence
+		FROM history_ledgers
+		WHERE sequence = (SELECT COALESCE(MAX(sequence), 0) FROM history_ledgers)
+	`)
+}
+
 // OldestOutdatedLedgers populates a slice of ints with the first million
 // outdated ledgers, based upon the provided `currentVersion` number
 func (q *Q) OldestOutdatedLedgers(dest interface{}, currentVersion int) error {

services/horizon/internal/db2/history/transaction.go

@@ -16,6 +16,17 @@ func (q *Q) TransactionByHash(dest interface{}, hash string) error {
 	return q.Get(dest, sql)
 }
 
+// TransactionsForLastXLedgers filters the query to only the last X ledgers worth of transactions.
+// Currently, we hard code the query to return the last 5 ledgers worth of transactions. In the
+// future this may be configurable.
+func (q *Q) TransactionsForLastXLedgers(currentSeq int32, dest interface{}) error {
+	return q.GetRaw(dest, `
+		SELECT min(fee_paid/operation_count),  mode() within group (order by fee_paid/operation_count)
+		FROM history_transactions
+		WHERE ledger_sequence > $1 AND ledger_sequence <= $2
+	`, currentSeq-5, currentSeq)
+}
+
 // Transactions provides a helper to filter rows from the `history_transactions`
 // table with pre-defined filters.  See `TransactionsQ` methods for the
 // available filters.

services/horizon/internal/init_web.go

@@ -148,6 +148,9 @@ func initWebActions(app *App) {
 		r.Get("/assets", AssetsAction{}.Handle)
 	}
 
+	// Network state related endpoints
+	r.Get("/operation_fee_stats", OperationFeeStatsAction{}.Handle)
+
 	// friendbot
 	if app.config.FriendbotURL != nil {
 		redirectFriendbot := func(w http.ResponseWriter, r *http.Request) {

services/horizon/internal/main_generated.go

@@ -64,6 +64,12 @@ func (action OffersByAccountAction) Handle(w http.ResponseWriter, r *http.Reques
 	ap.Execute(&action)
 }
 
+func (action OperationFeeStatsAction) Handle(w http.ResponseWriter, r *http.Request) {
+	ap := &action.Action
+	ap.Prepare(w, r)
+	ap.Execute(&action)
+}
+
 func (action OperationIndexAction) Handle(w http.ResponseWriter, r *http.Request) {
 	ap := &action.Action
 	ap.Prepare(w, r)

services/horizon/internal/operationfeestats/main.go

@@ -0,0 +1,46 @@
+// Package operationfeestats provides useful utilities concerning operation fee
+// stats within stellar,specifically as a central location to store a cached snapshot
+// of the state of network per operation fees and surge pricing. This package is
+// intended to be at the lowest levels of horizon's dependency tree, please keep
+// it free of dependencies to other horizon packages.
+package operationfeestats
+
+import (
+	"sync"
+)
+
+// State represents a snapshot of horizon's view of the state of operation fee's
+// on the network.
+type State struct {
+	Min         int64
+	Mode        int64
+	LastBaseFee int64
+	LastLedger  int64
+}
+
+// CurrentState returns the cached snapshot of operation fee state
+func CurrentState() State {
+	lock.RLock()
+	ret := current
+	lock.RUnlock()
+	return ret
+}
+
+// SetState updates the cached snapshot of the operation fee state
+func SetState(next State) {
+	lock.Lock()
+	// in case of one query taking longer than another, this makes
+	// sure we don't overwrite the latest fee stats with old stats
+	if current.LastLedger < next.LastLedger {
+		current = next
+	}
+	lock.Unlock()
+}
+
+// ResetState is used only for testing purposes
+func ResetState() {
+	current = State{}
+}
+
+var current State
+var lock sync.RWMutex