Unify transaction and event retention windows (#234)

* Unify transaction and event retention windows

Use a new, `unified history-retention-window` flag and mark the event/transaction window
configuration parameters as deprecated.

Until we remove the event/transaction flag, `history-retention-window` will be initialized to
the maximum of the three.

* Appease the linter

cmd/soroban-rpc/internal/config/main.go

@@ -41,6 +41,7 @@ type Config struct {
 	PreflightWorkerQueueSize                       uint
 	PreflightEnableDebug                           bool
 	SQLiteDBPath                                   string
+	HistoryRetentionWindow                         uint32
 	TransactionLedgerRetentionWindow               uint32
 	SorobanFeeStatsLedgerRetentionWindow           uint32
 	ClassicFeeStatsLedgerRetentionWindow           uint32
@@ -114,6 +115,13 @@ func (cfg *Config) SetValues(lookupEnv func(string) (string, bool)) error {
 		}
 	}
 
+	// Set to the maximum as a compromise until we deprecate the transaction/event flags
+	cfg.HistoryRetentionWindow = max(
+		cfg.HistoryRetentionWindow,
+		cfg.EventLedgerRetentionWindow,
+		cfg.TransactionLedgerRetentionWindow,
+	)
+
 	return nil
 }
 
@@ -126,6 +134,7 @@ func (cfg *Config) loadDefaults() error {
 			}
 		}
 	}
+	cfg.HistoryArchiveUserAgent = "soroban-rpc/" + Version
 	return nil
 }
 

cmd/soroban-rpc/internal/config/main_test.go

@@ -52,11 +52,9 @@ func TestConfigLoadDefaults(t *testing.T) {
 }
 
 func TestConfigExtendedUserAgent(t *testing.T) {
-	cfg := Config{
-		HistoryArchiveUserAgent: "Test",
-	}
+	var cfg Config
 	require.NoError(t, cfg.loadDefaults())
-	assert.Equal(t, "Test/123", cfg.ExtendedUserAgent("123"))
+	assert.Equal(t, "soroban-rpc/0.0.0/123", cfg.ExtendedUserAgent("123"))
 }
 
 func TestConfigLoadFlagsDefaultValuesOverrideExisting(t *testing.T) {

cmd/soroban-rpc/internal/config/options.go

@@ -13,11 +13,14 @@ import (
 
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/support/strutils"
-
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 )
 
-const defaultHTTPEndpoint = "localhost:8000"
+const (
+	// OneDayOfLedgers is (roughly) a 24 hour window of ledgers.
+	OneDayOfLedgers = 17280
+
+	defaultHTTPEndpoint = "localhost:8000"
+)
 
 // TODO: refactor and remove the linter exceptions
 //
@@ -159,7 +162,7 @@ func (cfg *Config) options() Options {
 					if v == "" || v == "." {
 						cwd, err := os.Getwd()
 						if err != nil {
-							return fmt.Errorf("unable to determine the current directory: %s", err)
+							return fmt.Errorf("unable to determine the current directory: %w", err)
 						}
 						v = cwd
 					}
@@ -168,7 +171,7 @@ func (cfg *Config) options() Options {
 				case nil:
 					cwd, err := os.Getwd()
 					if err != nil {
-						return fmt.Errorf("unable to determine the current directory: %s", err)
+						return fmt.Errorf("unable to determine the current directory: %w", err)
 					}
 					cfg.CaptiveCoreStoragePath = cwd
 					return nil
@@ -212,24 +215,36 @@ func (cfg *Config) options() Options {
 			ConfigKey:    &cfg.CheckpointFrequency,
 			DefaultValue: uint32(64),
 		},
+		{
+			Name: "history-retention-window",
+			Usage: fmt.Sprintf(
+				"configures history retention window for transactions and events, expressed in number of ledgers,"+
+					" the default value is %d which corresponds to about 24 hours of history",
+				OneDayOfLedgers),
+			ConfigKey:    &cfg.HistoryRetentionWindow,
+			DefaultValue: uint32(OneDayOfLedgers),
+			Validate:     positive,
+		},
+		// TODO: remove
 		{
 			Name: "event-retention-window",
 			Usage: fmt.Sprintf(
-				"configures the event retention window expressed in number of ledgers,"+
+				"(Deprecated, overidden by history-retention-window) configures the event retention window expressed in number of ledgers,"+
 					" the default value is %d which corresponds to about 24 hours of history",
-				ledgerbucketwindow.DefaultEventLedgerRetentionWindow),
+				OneDayOfLedgers),
 			ConfigKey:    &cfg.EventLedgerRetentionWindow,
-			DefaultValue: uint32(ledgerbucketwindow.DefaultEventLedgerRetentionWindow),
+			DefaultValue: uint32(OneDayOfLedgers),
 			Validate:     positive,
 		},
+		// TODO: remove
 		{
 			Name: "transaction-retention-window",
 			Usage: fmt.Sprintf(
-				"configures the transaction retention window expressed in number of ledgers,"+
+				"(Deprecated, overidden by history-retention-window) configures the transaction retention window expressed in number of ledgers,"+
 					" the default value is %d which corresponds to about 24 hours of history",
-				ledgerbucketwindow.OneDayOfLedgers),
+				OneDayOfLedgers),
 			ConfigKey:    &cfg.TransactionLedgerRetentionWindow,
-			DefaultValue: uint32(ledgerbucketwindow.OneDayOfLedgers),
+			DefaultValue: uint32(OneDayOfLedgers),
 			Validate:     positive,
 		},
 		{
@@ -504,7 +519,7 @@ func required(option *Option) error {
 		}
 	}
 
-	waysToSet := []string{}
+	var waysToSet []string
 	if option.Name != "" && option.Name != "-" {
 		waysToSet = append(waysToSet, fmt.Sprintf("specify --%s on the command line", option.Name))
 	}

cmd/soroban-rpc/internal/config/toml_test.go

@@ -92,7 +92,10 @@ func TestBasicTomlWriting(t *testing.T) {
 	// Note the newline at char 80. This also checks it adds a space after the
 	// comment when outputting multi-line comments, which go-toml does *not* do
 	// by default.
-	assert.Contains(t, out, "# configures the event retention window expressed in number of ledgers, the\n# default value is 17280 which corresponds to about 24 hours of history")
+	assert.Contains(t, out,
+		`# (Deprecated, overidden by history-retention-window) configures the event
+# retention window expressed in number of ledgers, the default value is 17280
+# which corresponds to about 24 hours of history`)
 }
 
 func TestRoundTrip(t *testing.T) {

cmd/soroban-rpc/internal/daemon/daemon.go

@@ -21,7 +21,6 @@ import (
 	"github.com/stellar/go/ingest/ledgerbackend"
 	supporthttp "github.com/stellar/go/support/http"
 	supportlog "github.com/stellar/go/support/log"
-	"github.com/stellar/go/support/ordered"
 	"github.com/stellar/go/support/storage"
 	"github.com/stellar/go/xdr"
 
@@ -31,7 +30,6 @@ import (
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util"
 )
@@ -207,23 +205,14 @@ func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon {
 		logger.WithError(err).Error("could not run ingestion. Retrying")
 	}
 
-	// Take the larger of (event retention, tx retention) and then the smaller
-	// of (tx retention, default event retention) if event retention wasn't
-	// specified, for some reason...?
-	maxRetentionWindow := ordered.Max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow)
-	if cfg.EventLedgerRetentionWindow <= 0 {
-		maxRetentionWindow = ordered.Min(
-			maxRetentionWindow,
-			ledgerbucketwindow.DefaultEventLedgerRetentionWindow)
-	}
 	ingestService := ingest.NewService(ingest.Config{
 		Logger: logger,
 		DB: db.NewReadWriter(
 			logger,
 			dbConn,
 			daemon,
 			maxLedgerEntryWriteBatchSize,
-			maxRetentionWindow,
+			cfg.HistoryRetentionWindow,
 			cfg.NetworkPassphrase,
 		),
 		EventStore:        eventStore,
@@ -305,7 +294,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow
 	eventStore := events.NewMemoryStore(
 		d,
 		cfg.NetworkPassphrase,
-		cfg.EventLedgerRetentionWindow,
+		cfg.HistoryRetentionWindow,
 	)
 	feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase)
 

cmd/soroban-rpc/internal/db/migration.go

@@ -183,7 +183,12 @@ func (g *guardedMigration) Rollback(ctx context.Context) error {
 
 func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) {
 	migrationName := "TransactionsTable"
-	factory := newTransactionTableMigration(ctx, logger.WithField("migration", migrationName), cfg.TransactionLedgerRetentionWindow, cfg.NetworkPassphrase)
+	factory := newTransactionTableMigration(
+		ctx,
+		logger.WithField("migration", migrationName),
+		cfg.HistoryRetentionWindow,
+		cfg.NetworkPassphrase,
+	)
 	m, err := newGuardedDataMigration(ctx, migrationName, factory, db)
 	if err != nil {
 		return nil, fmt.Errorf("creating guarded transaction migration: %w", err)

cmd/soroban-rpc/internal/integrationtest/health_test.go

@@ -6,16 +6,16 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/integrationtest/infrastructure"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 )
 
 func TestHealth(t *testing.T) {
 	test := infrastructure.NewTest(t, nil)
 	result, err := test.GetRPCHealth()
 	require.NoError(t, err)
 	assert.Equal(t, "healthy", result.Status)
-	assert.Equal(t, uint32(ledgerbucketwindow.OneDayOfLedgers), result.LedgerRetentionWindow)
+	assert.Equal(t, uint32(config.OneDayOfLedgers), result.LedgerRetentionWindow)
 	assert.Greater(t, result.OldestLedger, uint32(0))
 	assert.Greater(t, result.LatestLedger, uint32(0))
 	assert.GreaterOrEqual(t, result.LatestLedger, result.OldestLedger)

cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go

@@ -30,7 +30,6 @@ import (
 
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon"
-	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow"
 	"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods"
 )
 
@@ -321,8 +320,7 @@ func (vars rpcConfig) toMap() map[string]string {
 		"LOG_LEVEL":                      "debug",
 		"DB_PATH":                        vars.sqlitePath,
 		"INGESTION_TIMEOUT":              "10m",
-		"EVENT_LEDGER_RETENTION_WINDOW":  strconv.Itoa(ledgerbucketwindow.OneDayOfLedgers),
-		"TRANSACTION_RETENTION_WINDOW":   strconv.Itoa(ledgerbucketwindow.OneDayOfLedgers),
+		"HISTORY_RETENTION_WINDOW":       strconv.Itoa(config.OneDayOfLedgers),
 		"CHECKPOINT_FREQUENCY":           strconv.Itoa(checkpointFrequency),
 		"MAX_HEALTHY_LEDGER_LATENCY":     "10s",
 		"PREFLIGHT_ENABLE_DEBUG":         "true",
@@ -443,7 +441,6 @@ func (i *Test) createRPCDaemon(c rpcConfig) *daemon.Daemon {
 	}
 	require.NoError(i.t, cfg.SetValues(lookup))
 	require.NoError(i.t, cfg.Validate())
-	cfg.HistoryArchiveUserAgent = "soroban-rpc/" + config.Version
 
 	logger := supportlog.New()
 	logger.SetOutput(newTestLogWriter(i.t, `rpc="daemon" `))

cmd/soroban-rpc/internal/jsonrpc.go

@@ -137,10 +137,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler {
 		},
 	}
 
-	// While we transition from in-memory to database-oriented history storage,
-	// the on-disk (transaction) retention window will always be larger than the
-	// in-memory (events) one.
-	retentionWindow := cfg.TransactionLedgerRetentionWindow
+	retentionWindow := cfg.HistoryRetentionWindow
 
 	handlers := []struct {
 		methodName           string

cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go

@@ -20,18 +20,8 @@ type LedgerBucket[T any] struct {
 	BucketContent        T
 }
 
-// OneDayOfLedgers is (roughly) a 24 hour window of ledgers.
-const OneDayOfLedgers = 17280
-
-// DefaultEventLedgerRetentionWindow represents the max number of ledgers we
-// would like to keep an incoming event in memory.
-const DefaultEventLedgerRetentionWindow = OneDayOfLedgers
-
 // NewLedgerBucketWindow creates a new LedgerBucketWindow
 func NewLedgerBucketWindow[T any](retentionWindow uint32) *LedgerBucketWindow[T] {
-	if retentionWindow == 0 {
-		retentionWindow = DefaultEventLedgerRetentionWindow
-	}
 	return &LedgerBucketWindow[T]{
 		buckets: make([]LedgerBucket[T], 0, retentionWindow),
 	}

cmd/soroban-rpc/main.go

@@ -28,7 +28,6 @@ func main() {
 				fmt.Fprintln(os.Stderr, err)
 				os.Exit(1)
 			}
-			cfg.HistoryArchiveUserAgent = "soroban-rpc/" + config.Version
 			daemon.MustNew(&cfg, supportlog.New()).Run()
 		},
 	}