Cleanup old retention window and upgrade history-retention-window (#277)

* Cleanup old retention-windows and increase history-retention-window to 7 days * fix tests
stellar · Aug 30, 2024 · 7913265 · 7913265
1 parent 2daf023
commit 7913265
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 64 deletions.
diff --git a/cmd/soroban-rpc/internal/config/main.go b/cmd/soroban-rpc/internal/config/main.go
@@ -26,7 +26,6 @@ type Config struct {
 	CoreRequestTimeout                             time.Duration
 	DefaultEventsLimit                             uint
 	DefaultTransactionsLimit                       uint
-	EventLedgerRetentionWindow                     uint32
 	FriendbotURL                                   string
 	HistoryArchiveURLs                             []string
 	HistoryArchiveUserAgent                        string
@@ -42,7 +41,6 @@ type Config struct {
 	PreflightEnableDebug                           bool
 	SQLiteDBPath                                   string
 	HistoryRetentionWindow                         uint32
-	TransactionLedgerRetentionWindow               uint32
 	SorobanFeeStatsLedgerRetentionWindow           uint32
 	ClassicFeeStatsLedgerRetentionWindow           uint32
 	RequestBacklogGlobalQueueLimit                 uint
@@ -115,13 +113,6 @@ func (cfg *Config) SetValues(lookupEnv func(string) (string, bool)) error {
 		}
 	}
 
-	// Set to the maximum as a compromise until we deprecate the transaction/event flags
-	cfg.HistoryRetentionWindow = max(
-		cfg.HistoryRetentionWindow,
-		cfg.EventLedgerRetentionWindow,
-		cfg.TransactionLedgerRetentionWindow,
-	)
-
 	return nil
 }
 

diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go
@@ -218,32 +218,10 @@ func (cfg *Config) options() Options {
 			Name: "history-retention-window",
 			Usage: fmt.Sprintf(
 				"configures history retention window for transactions and events, expressed in number of ledgers,"+
-					" the default value is %d which corresponds to about 24 hours of history",
-				OneDayOfLedgers),
+					" the default value is %d which corresponds to about 7 days of history",
+				SevenDayOfLedgers),
 			ConfigKey:    &cfg.HistoryRetentionWindow,
-			DefaultValue: uint32(OneDayOfLedgers),
-			Validate:     positive,
-		},
-		// TODO: remove
-		{
-			Name: "event-retention-window",
-			Usage: fmt.Sprintf(
-				"(Deprecated, overidden by history-retention-window) configures the event retention window expressed in number of ledgers,"+
-					" the default value is %d which corresponds to about 24 hours of history",
-				OneDayOfLedgers),
-			ConfigKey:    &cfg.EventLedgerRetentionWindow,
-			DefaultValue: uint32(OneDayOfLedgers),
-			Validate:     positive,
-		},
-		// TODO: remove
-		{
-			Name: "transaction-retention-window",
-			Usage: fmt.Sprintf(
-				"(Deprecated, overidden by history-retention-window) configures the transaction retention window expressed in number of ledgers,"+
-					" the default value is %d which corresponds to about 24 hours of history",
-				OneDayOfLedgers),
-			ConfigKey:    &cfg.TransactionLedgerRetentionWindow,
-			DefaultValue: uint32(OneDayOfLedgers),
+			DefaultValue: uint32(SevenDayOfLedgers),
 			Validate:     positive,
 		},
 		{

diff --git a/cmd/soroban-rpc/internal/config/toml_test.go b/cmd/soroban-rpc/internal/config/toml_test.go
@@ -93,9 +93,9 @@ func TestBasicTomlWriting(t *testing.T) {
 	// comment when outputting multi-line comments, which go-toml does *not* do
 	// by default.
 	assert.Contains(t, out,
-		`# (Deprecated, overidden by history-retention-window) configures the event
-# retention window expressed in number of ledgers, the default value is 17280
-# which corresponds to about 24 hours of history`)
+		`# configures history retention window for transactions and events, expressed in
+# number of ledgers, the default value is 120960 which corresponds to about 7
+# days of history`)
 }
 
 func TestRoundTrip(t *testing.T) {

diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go
@@ -163,11 +163,11 @@ type ReadWriterMetrics struct {
 }
 
 type readWriter struct {
-	log                   *log.Entry
-	db                    *DB
-	maxBatchSize          int
-	ledgerRetentionWindow uint32
-	passphrase            string
+	log                    *log.Entry
+	db                     *DB
+	maxBatchSize           int
+	historyRetentionWindow uint32
+	passphrase             string
 
 	metrics ReadWriterMetrics
 }
@@ -181,7 +181,7 @@ func NewReadWriter(
 	db *DB,
 	daemon interfaces.Daemon,
 	maxBatchSize int,
-	ledgerRetentionWindow uint32,
+	historyRetentionWindow uint32,
 	networkPassphrase string,
 ) ReadWriter {
 	// a metric for measuring latency of transaction store operations
@@ -203,11 +203,11 @@ func NewReadWriter(
 	daemon.MetricsRegistry().MustRegister(txDurationMetric, txCountMetric)
 
 	return &readWriter{
-		log:                   log,
-		db:                    db,
-		maxBatchSize:          maxBatchSize,
-		ledgerRetentionWindow: ledgerRetentionWindow,
-		passphrase:            networkPassphrase,
+		log:                    log,
+		db:                     db,
+		maxBatchSize:           maxBatchSize,
+		historyRetentionWindow: historyRetentionWindow,
+		passphrase:             networkPassphrase,
 		metrics: ReadWriterMetrics{
 			TxIngestDuration: txDurationMetric.With(prometheus.Labels{"operation": "ingest"}),
 			TxCount:          txCountMetric,
@@ -234,10 +234,10 @@ func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) {
 			_, err := db.ExecRaw(ctx, "PRAGMA wal_checkpoint(TRUNCATE)")
 			return err
 		},
-		tx:                    txSession,
-		stmtCache:             stmtCache,
-		ledgerRetentionWindow: rw.ledgerRetentionWindow,
-		ledgerWriter:          ledgerWriter{stmtCache: stmtCache},
+		tx:                     txSession,
+		stmtCache:              stmtCache,
+		historyRetentionWindow: rw.historyRetentionWindow,
+		ledgerWriter:           ledgerWriter{stmtCache: stmtCache},
 		ledgerEntryWriter: ledgerEntryWriter{
 			stmtCache:               stmtCache,
 			buffer:                  xdr.NewEncodingBuffer(),
@@ -266,15 +266,15 @@ func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) {
 }
 
 type writeTx struct {
-	globalCache           *dbCache
-	postCommit            func() error
-	tx                    db.SessionInterface
-	stmtCache             *sq.StmtCache
-	ledgerEntryWriter     ledgerEntryWriter
-	ledgerWriter          ledgerWriter
-	txWriter              transactionHandler
-	eventWriter           eventHandler
-	ledgerRetentionWindow uint32
+	globalCache            *dbCache
+	postCommit             func() error
+	tx                     db.SessionInterface
+	stmtCache              *sq.StmtCache
+	ledgerEntryWriter      ledgerEntryWriter
+	ledgerWriter           ledgerWriter
+	txWriter               transactionHandler
+	eventWriter            eventHandler
+	historyRetentionWindow uint32
 }
 
 func (w writeTx) LedgerEntryWriter() LedgerEntryWriter {
@@ -298,14 +298,14 @@ func (w writeTx) Commit(ledgerSeq uint32) error {
 		return err
 	}
 
-	if err := w.ledgerWriter.trimLedgers(ledgerSeq, w.ledgerRetentionWindow); err != nil {
+	if err := w.ledgerWriter.trimLedgers(ledgerSeq, w.historyRetentionWindow); err != nil {
 		return err
 	}
-	if err := w.txWriter.trimTransactions(ledgerSeq, w.ledgerRetentionWindow); err != nil {
+	if err := w.txWriter.trimTransactions(ledgerSeq, w.historyRetentionWindow); err != nil {
 		return err
 	}
 
-	if err := w.eventWriter.trimEvents(ledgerSeq, w.ledgerRetentionWindow); err != nil {
+	if err := w.eventWriter.trimEvents(ledgerSeq, w.historyRetentionWindow); err != nil {
 		return err
 	}