diff --git a/cmd/soroban-rpc/internal/config/main.go b/cmd/soroban-rpc/internal/config/main.go index 1321feb6..4282d824 100644 --- a/cmd/soroban-rpc/internal/config/main.go +++ b/cmd/soroban-rpc/internal/config/main.go @@ -41,6 +41,7 @@ type Config struct { PreflightWorkerQueueSize uint PreflightEnableDebug bool SQLiteDBPath string + HistoryRetentionWindow uint32 TransactionLedgerRetentionWindow uint32 SorobanFeeStatsLedgerRetentionWindow uint32 ClassicFeeStatsLedgerRetentionWindow uint32 @@ -114,6 +115,13 @@ func (cfg *Config) SetValues(lookupEnv func(string) (string, bool)) error { } } + // Set to the maximum as a compromise until we deprecate the transaction/event flags + cfg.HistoryRetentionWindow = max( + cfg.HistoryRetentionWindow, + cfg.EventLedgerRetentionWindow, + cfg.TransactionLedgerRetentionWindow, + ) + return nil } @@ -126,6 +134,7 @@ func (cfg *Config) loadDefaults() error { } } } + cfg.HistoryArchiveUserAgent = "soroban-rpc/" + Version return nil } diff --git a/cmd/soroban-rpc/internal/config/main_test.go b/cmd/soroban-rpc/internal/config/main_test.go index df006b7e..c510c1bd 100644 --- a/cmd/soroban-rpc/internal/config/main_test.go +++ b/cmd/soroban-rpc/internal/config/main_test.go @@ -52,11 +52,9 @@ func TestConfigLoadDefaults(t *testing.T) { } func TestConfigExtendedUserAgent(t *testing.T) { - cfg := Config{ - HistoryArchiveUserAgent: "Test", - } + var cfg Config require.NoError(t, cfg.loadDefaults()) - assert.Equal(t, "Test/123", cfg.ExtendedUserAgent("123")) + assert.Equal(t, "soroban-rpc/0.0.0/123", cfg.ExtendedUserAgent("123")) } func TestConfigLoadFlagsDefaultValuesOverrideExisting(t *testing.T) { diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index e4838ab5..9d14084a 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -13,11 +13,14 @@ import ( "github.com/stellar/go/network" "github.com/stellar/go/support/strutils" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) -const defaultHTTPEndpoint = "localhost:8000" +const ( + // OneDayOfLedgers is (roughly) a 24 hour window of ledgers. + OneDayOfLedgers = 17280 + + defaultHTTPEndpoint = "localhost:8000" +) // TODO: refactor and remove the linter exceptions // @@ -159,7 +162,7 @@ func (cfg *Config) options() Options { if v == "" || v == "." { cwd, err := os.Getwd() if err != nil { - return fmt.Errorf("unable to determine the current directory: %s", err) + return fmt.Errorf("unable to determine the current directory: %w", err) } v = cwd } @@ -168,7 +171,7 @@ func (cfg *Config) options() Options { case nil: cwd, err := os.Getwd() if err != nil { - return fmt.Errorf("unable to determine the current directory: %s", err) + return fmt.Errorf("unable to determine the current directory: %w", err) } cfg.CaptiveCoreStoragePath = cwd return nil @@ -212,24 +215,36 @@ func (cfg *Config) options() Options { ConfigKey: &cfg.CheckpointFrequency, DefaultValue: uint32(64), }, + { + Name: "history-retention-window", + Usage: fmt.Sprintf( + "configures history retention window for transactions and events, expressed in number of ledgers,"+ + " the default value is %d which corresponds to about 24 hours of history", + OneDayOfLedgers), + ConfigKey: &cfg.HistoryRetentionWindow, + DefaultValue: uint32(OneDayOfLedgers), + Validate: positive, + }, + // TODO: remove { Name: "event-retention-window", Usage: fmt.Sprintf( - "configures the event retention window expressed in number of ledgers,"+ + "(Deprecated, overidden by history-retention-window) configures the event retention window expressed in number of ledgers,"+ " the default value is %d which corresponds to about 24 hours of history", - ledgerbucketwindow.DefaultEventLedgerRetentionWindow), + OneDayOfLedgers), ConfigKey: &cfg.EventLedgerRetentionWindow, - DefaultValue: uint32(ledgerbucketwindow.DefaultEventLedgerRetentionWindow), + DefaultValue: uint32(OneDayOfLedgers), Validate: positive, }, + // TODO: remove { Name: "transaction-retention-window", Usage: fmt.Sprintf( - "configures the transaction retention window expressed in number of ledgers,"+ + "(Deprecated, overidden by history-retention-window) configures the transaction retention window expressed in number of ledgers,"+ " the default value is %d which corresponds to about 24 hours of history", - ledgerbucketwindow.OneDayOfLedgers), + OneDayOfLedgers), ConfigKey: &cfg.TransactionLedgerRetentionWindow, - DefaultValue: uint32(ledgerbucketwindow.OneDayOfLedgers), + DefaultValue: uint32(OneDayOfLedgers), Validate: positive, }, { @@ -504,7 +519,7 @@ func required(option *Option) error { } } - waysToSet := []string{} + var waysToSet []string if option.Name != "" && option.Name != "-" { waysToSet = append(waysToSet, fmt.Sprintf("specify --%s on the command line", option.Name)) } diff --git a/cmd/soroban-rpc/internal/config/toml_test.go b/cmd/soroban-rpc/internal/config/toml_test.go index 0b438996..ae762567 100644 --- a/cmd/soroban-rpc/internal/config/toml_test.go +++ b/cmd/soroban-rpc/internal/config/toml_test.go @@ -92,7 +92,10 @@ func TestBasicTomlWriting(t *testing.T) { // Note the newline at char 80. This also checks it adds a space after the // comment when outputting multi-line comments, which go-toml does *not* do // by default. - assert.Contains(t, out, "# configures the event retention window expressed in number of ledgers, the\n# default value is 17280 which corresponds to about 24 hours of history") + assert.Contains(t, out, + `# (Deprecated, overidden by history-retention-window) configures the event +# retention window expressed in number of ledgers, the default value is 17280 +# which corresponds to about 24 hours of history`) } func TestRoundTrip(t *testing.T) { diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index 5ca3004b..ab8ec8a0 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -21,7 +21,6 @@ import ( "github.com/stellar/go/ingest/ledgerbackend" supporthttp "github.com/stellar/go/support/http" supportlog "github.com/stellar/go/support/log" - "github.com/stellar/go/support/ordered" "github.com/stellar/go/support/storage" "github.com/stellar/go/xdr" @@ -31,7 +30,6 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/preflight" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util" ) @@ -207,15 +205,6 @@ func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon { logger.WithError(err).Error("could not run ingestion. Retrying") } - // Take the larger of (event retention, tx retention) and then the smaller - // of (tx retention, default event retention) if event retention wasn't - // specified, for some reason...? - maxRetentionWindow := ordered.Max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow) - if cfg.EventLedgerRetentionWindow <= 0 { - maxRetentionWindow = ordered.Min( - maxRetentionWindow, - ledgerbucketwindow.DefaultEventLedgerRetentionWindow) - } ingestService := ingest.NewService(ingest.Config{ Logger: logger, DB: db.NewReadWriter( @@ -223,7 +212,7 @@ func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon { dbConn, daemon, maxLedgerEntryWriteBatchSize, - maxRetentionWindow, + cfg.HistoryRetentionWindow, cfg.NetworkPassphrase, ), EventStore: eventStore, @@ -305,7 +294,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow eventStore := events.NewMemoryStore( d, cfg.NetworkPassphrase, - cfg.EventLedgerRetentionWindow, + cfg.HistoryRetentionWindow, ) feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase) diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go index 74a88f87..dbe07ab5 100644 --- a/cmd/soroban-rpc/internal/db/migration.go +++ b/cmd/soroban-rpc/internal/db/migration.go @@ -183,7 +183,12 @@ func (g *guardedMigration) Rollback(ctx context.Context) error { func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) { migrationName := "TransactionsTable" - factory := newTransactionTableMigration(ctx, logger.WithField("migration", migrationName), cfg.TransactionLedgerRetentionWindow, cfg.NetworkPassphrase) + factory := newTransactionTableMigration( + ctx, + logger.WithField("migration", migrationName), + cfg.HistoryRetentionWindow, + cfg.NetworkPassphrase, + ) m, err := newGuardedDataMigration(ctx, migrationName, factory, db) if err != nil { return nil, fmt.Errorf("creating guarded transaction migration: %w", err) diff --git a/cmd/soroban-rpc/internal/integrationtest/health_test.go b/cmd/soroban-rpc/internal/integrationtest/health_test.go index 2fe9ab86..c06639cd 100644 --- a/cmd/soroban-rpc/internal/integrationtest/health_test.go +++ b/cmd/soroban-rpc/internal/integrationtest/health_test.go @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/integrationtest/infrastructure" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) func TestHealth(t *testing.T) { @@ -15,7 +15,7 @@ func TestHealth(t *testing.T) { result, err := test.GetRPCHealth() require.NoError(t, err) assert.Equal(t, "healthy", result.Status) - assert.Equal(t, uint32(ledgerbucketwindow.OneDayOfLedgers), result.LedgerRetentionWindow) + assert.Equal(t, uint32(config.OneDayOfLedgers), result.LedgerRetentionWindow) assert.Greater(t, result.OldestLedger, uint32(0)) assert.Greater(t, result.LatestLedger, uint32(0)) assert.GreaterOrEqual(t, result.LatestLedger, result.OldestLedger) diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index f8996e12..c839a553 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -30,7 +30,6 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods" ) @@ -321,8 +320,7 @@ func (vars rpcConfig) toMap() map[string]string { "LOG_LEVEL": "debug", "DB_PATH": vars.sqlitePath, "INGESTION_TIMEOUT": "10m", - "EVENT_LEDGER_RETENTION_WINDOW": strconv.Itoa(ledgerbucketwindow.OneDayOfLedgers), - "TRANSACTION_RETENTION_WINDOW": strconv.Itoa(ledgerbucketwindow.OneDayOfLedgers), + "HISTORY_RETENTION_WINDOW": strconv.Itoa(config.OneDayOfLedgers), "CHECKPOINT_FREQUENCY": strconv.Itoa(checkpointFrequency), "MAX_HEALTHY_LEDGER_LATENCY": "10s", "PREFLIGHT_ENABLE_DEBUG": "true", @@ -443,7 +441,6 @@ func (i *Test) createRPCDaemon(c rpcConfig) *daemon.Daemon { } require.NoError(i.t, cfg.SetValues(lookup)) require.NoError(i.t, cfg.Validate()) - cfg.HistoryArchiveUserAgent = "soroban-rpc/" + config.Version logger := supportlog.New() logger.SetOutput(newTestLogWriter(i.t, `rpc="daemon" `)) diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index aede6b93..9236a4a4 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -137,10 +137,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { }, } - // While we transition from in-memory to database-oriented history storage, - // the on-disk (transaction) retention window will always be larger than the - // in-memory (events) one. - retentionWindow := cfg.TransactionLedgerRetentionWindow + retentionWindow := cfg.HistoryRetentionWindow handlers := []struct { methodName string diff --git a/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go b/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go index fe0eb58e..0f3ca5e2 100644 --- a/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go +++ b/cmd/soroban-rpc/internal/ledgerbucketwindow/ledgerbucketwindow.go @@ -20,18 +20,8 @@ type LedgerBucket[T any] struct { BucketContent T } -// OneDayOfLedgers is (roughly) a 24 hour window of ledgers. -const OneDayOfLedgers = 17280 - -// DefaultEventLedgerRetentionWindow represents the max number of ledgers we -// would like to keep an incoming event in memory. -const DefaultEventLedgerRetentionWindow = OneDayOfLedgers - // NewLedgerBucketWindow creates a new LedgerBucketWindow func NewLedgerBucketWindow[T any](retentionWindow uint32) *LedgerBucketWindow[T] { - if retentionWindow == 0 { - retentionWindow = DefaultEventLedgerRetentionWindow - } return &LedgerBucketWindow[T]{ buckets: make([]LedgerBucket[T], 0, retentionWindow), } diff --git a/cmd/soroban-rpc/main.go b/cmd/soroban-rpc/main.go index 84fa9116..e2150451 100644 --- a/cmd/soroban-rpc/main.go +++ b/cmd/soroban-rpc/main.go @@ -28,7 +28,6 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - cfg.HistoryArchiveUserAgent = "soroban-rpc/" + config.Version daemon.MustNew(&cfg, supportlog.New()).Run() }, }