Add getFeeStats method

cmd/soroban-rpc/internal/config/config.go

@@ -20,47 +20,51 @@ type Config struct {
 	CaptiveCoreConfigPath  string
 	CaptiveCoreHTTPPort    uint
 
-	Endpoint                                    string
-	AdminEndpoint                               string
-	CheckpointFrequency                         uint32
-	CoreRequestTimeout                          time.Duration
-	DefaultEventsLimit                          uint
-	EventLedgerRetentionWindow                  uint32
-	FriendbotURL                                string
-	HistoryArchiveURLs                          []string
-	HistoryArchiveUserAgent                     string
-	IngestionTimeout                            time.Duration
-	LogFormat                                   LogFormat
-	LogLevel                                    logrus.Level
-	MaxEventsLimit                              uint
-	MaxHealthyLedgerLatency                     time.Duration
-	NetworkPassphrase                           string
-	PreflightWorkerCount                        uint
-	PreflightWorkerQueueSize                    uint
-	PreflightEnableDebug                        bool
-	SQLiteDBPath                                string
-	TransactionLedgerRetentionWindow            uint32
-	RequestBacklogGlobalQueueLimit              uint
-	RequestBacklogGetHealthQueueLimit           uint
-	RequestBacklogGetEventsQueueLimit           uint
-	RequestBacklogGetNetworkQueueLimit          uint
-	RequestBacklogGetVersionInfoQueueLimit      uint
-	RequestBacklogGetLatestLedgerQueueLimit     uint
-	RequestBacklogGetLedgerEntriesQueueLimit    uint
-	RequestBacklogGetTransactionQueueLimit      uint
-	RequestBacklogSendTransactionQueueLimit     uint
-	RequestBacklogSimulateTransactionQueueLimit uint
-	RequestExecutionWarningThreshold            time.Duration
-	MaxRequestExecutionDuration                 time.Duration
-	MaxGetHealthExecutionDuration               time.Duration
-	MaxGetEventsExecutionDuration               time.Duration
-	MaxGetNetworkExecutionDuration              time.Duration
-	MaxGetVersionInfoExecutionDuration          time.Duration
-	MaxGetLatestLedgerExecutionDuration         time.Duration
-	MaxGetLedgerEntriesExecutionDuration        time.Duration
-	MaxGetTransactionExecutionDuration          time.Duration
-	MaxSendTransactionExecutionDuration         time.Duration
-	MaxSimulateTransactionExecutionDuration     time.Duration
+	Endpoint                                       string
+	AdminEndpoint                                  string
+	CheckpointFrequency                            uint32
+	CoreRequestTimeout                             time.Duration
+	DefaultEventsLimit                             uint
+	EventLedgerRetentionWindow                     uint32
+	FriendbotURL                                   string
+	HistoryArchiveURLs                             []string
+	HistoryArchiveUserAgent                        string
+	IngestionTimeout                               time.Duration
+	LogFormat                                      LogFormat
+	LogLevel                                       logrus.Level
+	MaxEventsLimit                                 uint
+	MaxHealthyLedgerLatency                        time.Duration
+	NetworkPassphrase                              string
+	PreflightWorkerCount                           uint
+	PreflightWorkerQueueSize                       uint
+	PreflightEnableDebug                           bool
+	SQLiteDBPath                                   string
+	TransactionLedgerRetentionWindow               uint32
+	SorobanFeeStatsLedgerRetentionWindow           uint32
+	ClassicFeeStatsLedgerRetentionWindow           uint32
+	RequestBacklogGlobalQueueLimit                 uint
+	RequestBacklogGetHealthQueueLimit              uint
+	RequestBacklogGetEventsQueueLimit              uint
+	RequestBacklogGetNetworkQueueLimit             uint
+	RequestBacklogGetVersionInfoQueueLimit         uint
+	RequestBacklogGetLatestLedgerQueueLimit        uint
+	RequestBacklogGetLedgerEntriesQueueLimit       uint
+	RequestBacklogGetTransactionQueueLimit         uint
+	RequestBacklogSendTransactionQueueLimit        uint
+	RequestBacklogSimulateTransactionQueueLimit    uint
+	RequestBacklogGetFeeStatsTransactionQueueLimit uint
+	RequestExecutionWarningThreshold               time.Duration
+	MaxRequestExecutionDuration                    time.Duration
+	MaxGetHealthExecutionDuration                  time.Duration
+	MaxGetEventsExecutionDuration                  time.Duration
+	MaxGetNetworkExecutionDuration                 time.Duration
+	MaxGetVersionInfoExecutionDuration             time.Duration
+	MaxGetLatestLedgerExecutionDuration            time.Duration
+	MaxGetLedgerEntriesExecutionDuration           time.Duration
+	MaxGetTransactionExecutionDuration             time.Duration
+	MaxSendTransactionExecutionDuration            time.Duration
+	MaxSimulateTransactionExecutionDuration        time.Duration
+	MaxGetFeeStatsExecutionDuration                time.Duration
 
 	// We memoize these, so they bind to pflags correctly
 	optionsCache *ConfigOptions

cmd/soroban-rpc/internal/config/options.go

@@ -225,6 +225,20 @@ func (cfg *Config) options() ConfigOptions {
 			DefaultValue: uint32(1440),
 			Validate:     positive,
 		},
+		{
+			Name:         "classic-fee-stats-retention-window",
+			Usage:        "configures classic fee stats retention window expressed in number of ledgers",
+			ConfigKey:    &cfg.ClassicFeeStatsLedgerRetentionWindow,
+			DefaultValue: uint32(5),
+			Validate:     positive,
+		},
+		{
+			Name:         "soroban-fee-stats-retention-window",
+			Usage:        "configures soroban inclusion fee stats retention window expressed in number of ledgers",
+			ConfigKey:    &cfg.TransactionLedgerRetentionWindow,
+			DefaultValue: uint32(50),
+			Validate:     positive,
+		},
 		{
 			Name:         "max-events-limit",
 			Usage:        "Maximum amount of events allowed in a single getEvents response",
@@ -344,6 +358,13 @@ func (cfg *Config) options() ConfigOptions {
 			DefaultValue: uint(100),
 			Validate:     positive,
 		},
+		{
+			TomlKey:      strutils.KebabToConstantCase("request-backlog-get-fee-stats-queue-limit"),
+			Usage:        "Maximum number of outstanding GetFeeStats requests",
+			ConfigKey:    &cfg.RequestBacklogGetFeeStatsTransactionQueueLimit,
+			DefaultValue: uint(100),
+			Validate:     positive,
+		},
 		{
 			TomlKey:      strutils.KebabToConstantCase("request-execution-warning-threshold"),
 			Usage:        "The request execution warning threshold is the predetermined maximum duration of time that a request can take to be processed before a warning would be generated",
@@ -410,6 +431,12 @@ func (cfg *Config) options() ConfigOptions {
 			ConfigKey:    &cfg.MaxSimulateTransactionExecutionDuration,
 			DefaultValue: 15 * time.Second,
 		},
+		{
+			TomlKey:      strutils.KebabToConstantCase("max-get-fee-stats-execution-duration"),
+			Usage:        "The maximum duration of time allowed for processing a getFeeStats request. When that time elapses, the rpc server would return -32001 and abort the request's execution",
+			ConfigKey:    &cfg.MaxGetFeeStatsExecutionDuration,
+			DefaultValue: 5 * time.Second,
+		},
 	}
 	return *cfg.optionsCache
 }

cmd/soroban-rpc/internal/daemon/daemon.go

@@ -26,8 +26,8 @@
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events"
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/transactions"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util"
@@ -195,12 +195,13 @@
 		cfg.NetworkPassphrase,
 		cfg.TransactionLedgerRetentionWindow,
 	)
+	feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase)
 
 	// initialize the stores using what was on the DB
 	readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout)
 	defer cancelReadTxMeta()
 	// NOTE: We could optimize this to avoid unnecessary ingestion calls
-	//       (the range of txmetads can be larger than the store retention windows)
+	//       (the range of txmetas can be larger than the individual store retention windows)
 	//       but it's probably not worth the pain.
 	var initialSeq uint32
 	var currentSeq uint32
@@ -222,6 +223,9 @@
 		if err := transactionStore.IngestTransactions(txmeta); err != nil {
 			logger.WithError(err).Fatal("could not initialize transaction memory store")
 		}
+		if err := feewindows.IngestFees(txmeta); err != nil {
+			logger.WithError(err).Fatal("could not initialize fee stats")
+		}
 		return nil
 	})
 	if err != nil {
@@ -236,12 +240,7 @@
 	onIngestionRetry := func(err error, dur time.Duration) {
 		logger.WithError(err).Error("could not run ingestion. Retrying")
 	}
-	maxRetentionWindow := cfg.EventLedgerRetentionWindow
-	if cfg.TransactionLedgerRetentionWindow > maxRetentionWindow {
-		maxRetentionWindow = cfg.TransactionLedgerRetentionWindow
-	} else if cfg.EventLedgerRetentionWindow == 0 && cfg.TransactionLedgerRetentionWindow > ledgerbucketwindow.DefaultEventLedgerRetentionWindow {
-		maxRetentionWindow = ledgerbucketwindow.DefaultEventLedgerRetentionWindow
-	}
+	maxRetentionWindow := max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow, cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow)
 	ingestService := ingest.NewService(ingest.Config{
 		Logger:            logger,
 		DB:                db.NewReadWriter(dbConn, maxLedgerEntryWriteBatchSize, maxRetentionWindow),
@@ -253,6 +252,7 @@
 		Timeout:           cfg.IngestionTimeout,
 		OnIngestionRetry:  onIngestionRetry,
 		Daemon:            daemon,
+		FeeWindows:        feewindows,
 	})
 
 	ledgerEntryReader := db.NewLedgerEntryReader(dbConn)

cmd/soroban-rpc/internal/feewindow/feewindow.go

@@ -0,0 +1,178 @@
+package feewindow
+
+import (
+	"io"
+	"slices"
+	"sync"
+
+	"github.com/stellar/go/ingest"
+	"github.com/stellar/go/xdr"
+
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
+)
+
+type FeeDistribution struct {
+	Max   uint64
+	Min   uint64
+	Mode  uint64
+	P10   uint64
+	P20   uint64
+	P30   uint64
+	P40   uint64
+	P50   uint64
+	P60   uint64
+	P70   uint64
+	P80   uint64
+	P90   uint64
+	P95   uint64
+	P99   uint64
+	Count uint64
+}
+
+type FeeWindow struct {
+	lock          sync.RWMutex
+	totalFeeCount uint64
+	feesPerLedger *ledgerbucketwindow.LedgerBucketWindow[[]uint64]
+	distribution  FeeDistribution
+}
+
+func NewFeeWindow(retentionWindow uint32) *FeeWindow {
+	window := ledgerbucketwindow.NewLedgerBucketWindow[[]uint64](retentionWindow)
+	return &FeeWindow{
+		totalFeeCount: 0,
+		feesPerLedger: window,
+	}
+}
+
+func (fw *FeeWindow) AppendLedgerFees(fees ledgerbucketwindow.LedgerBucket[[]uint64]) error {
+	fw.lock.Lock()
+	defer fw.lock.Unlock()
+	_, err := fw.feesPerLedger.Append(fees)
+	if err != nil {
+		return err
+	}
+
+	var allFees []uint64
+	for i := uint32(0); i < fw.feesPerLedger.Len(); i++ {
+		allFees = append(allFees, fw.feesPerLedger.Get(i).BucketContent...)
+	}
+	fw.distribution = computeFeeDistribution(allFees)
+
+	return nil
+}
+
+func computeFeeDistribution(fees []uint64) FeeDistribution {
+	if len(fees) == 0 {
+		return FeeDistribution{}
+	}
+	slices.Sort(fees)
+	mode := fees[0]
+	lastVal := fees[0]
+	maxRepetitions := 0
+	localRepetitions := 0
+	for i := 1; i < len(fees); i++ {
+		if fees[i] == lastVal {
+			localRepetitions += 1
+			continue
+		}
+
+		// new cluster of values
+
+		if localRepetitions > maxRepetitions {
+			maxRepetitions = localRepetitions
+			mode = fees[i]
+		}
+		lastVal = fees[i]
+		localRepetitions = 0
+	}
+	count := uint64(len(fees))
+	// nearest-rank percentile
+	percentile := func(p uint64) uint64 {
+		// ceiling(p*count/100)
+		kth := ((p * count) + 100 - 1) / 100
+		return fees[kth-1]
+	}
+	return FeeDistribution{
+		Max:   fees[len(fees)-1],
+		Min:   fees[0],
+		Mode:  mode,
+		P10:   percentile(10),
+		P20:   percentile(20),
+		P30:   percentile(30),
+		P40:   percentile(40),
+		P50:   percentile(50),
+		P60:   percentile(60),
+		P70:   percentile(70),
+		P80:   percentile(80),
+		P90:   percentile(90),
+		P95:   percentile(95),
+		P99:   percentile(99),
+		Count: count,
+	}
+}
+
+func (fw *FeeWindow) GetFeeDistribution() FeeDistribution {
+	fw.lock.RLock()
+	defer fw.lock.RUnlock()
+	return fw.distribution
+}
+
+type FeeWindows struct {
+	SorobanInclusionFeeWindow *FeeWindow
+	ClassicFeeWindow          *FeeWindow
+	networkPassPhrase         string
+}
+
+func NewFeeWindows(classicRetention uint32, sorobanRetetion uint32, networkPassPhrase string) *FeeWindows {
+	return &FeeWindows{
+		SorobanInclusionFeeWindow: NewFeeWindow(sorobanRetetion),
+		ClassicFeeWindow:          NewFeeWindow(classicRetention),
+		networkPassPhrase:         networkPassPhrase,
+	}
+}
+
+func (fw *FeeWindows) IngestFees(meta xdr.LedgerCloseMeta) error {
+	reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(fw.networkPassPhrase, meta)
+	if err != nil {
+		return err
+	}
+	var sorobanInclusionFees []uint64
+	var classicFees []uint64
+	for {
+		tx, err := reader.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		ops := tx.Envelope.Operations()
+		if len(ops) == 1 {
+			switch ops[0].Body.Type {
+			case xdr.OperationTypeInvokeHostFunction, xdr.OperationTypeExtendFootprintTtl, xdr.OperationTypeRestoreFootprint:
+				if tx.Envelope.V1 == nil || tx.Envelope.V1.Tx.Ext.SorobanData != nil {
+					// this shouldn't happen
+					continue
+				}
+				inclusionFee := uint64(tx.Envelope.V1.Tx.Fee) - uint64(tx.Envelope.V1.Tx.Ext.SorobanData.ResourceFee)
+				classicFees = append(sorobanInclusionFees, inclusionFee)
+				continue
+			}
+		}
+		classicFees = append(classicFees, uint64(tx.Envelope.Fee()))
+
+	}
+	bucket := ledgerbucketwindow.LedgerBucket[[]uint64]{
+		LedgerSeq:            meta.LedgerSequence(),
+		LedgerCloseTimestamp: meta.LedgerCloseTime(),
+		BucketContent:        classicFees,
+	}
+	if err := fw.ClassicFeeWindow.AppendLedgerFees(bucket); err != nil {
+		return err
+	}
+	bucket.BucketContent = sorobanInclusionFees
+	if err := fw.SorobanInclusionFeeWindow.AppendLedgerFees(bucket); err != nil {
+		return err
+	}
+	return nil
+}