From 6269006827c42f006dac9e354099f9e8f936f3f4 Mon Sep 17 00:00:00 2001 From: Tamir Sen Date: Thu, 27 Feb 2020 22:31:30 +0100 Subject: [PATCH 1/3] Fix horizon db reap command When the db reap command is called, `ledger.CurrentState()` returns an uninitialized struct where the latest history ledger sequence is set to 0. Consequently, the reap system always concludes that there are no ledgers preceding the cutoff point. To fix this bug we need to call `app.UpdateLedgerState()` to ensure that ledger.CurrentState() is properly initialized. Another problem with the reap command is that it does not delete from all history tables (e.g. history_trades). This commit fixes this issue by calling `DeleteRangeAll()` which is used by the ingestion system to clear history tables prior to running a reingest range command. --- services/horizon/CHANGELOG.md | 5 +++++ services/horizon/cmd/db.go | 4 +++- services/horizon/internal/reap/main.go | 7 ++++--- services/horizon/internal/reap/system.go | 24 +++--------------------- 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 9208d78c9b..e5eaba840e 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -3,6 +3,11 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## 1.1.0 + +* Fix `horizon db reap` bug which caused the command to exit without deleting any history table rows. +* The horizon reap system now also deletes rows from `history_effects`. Previously, the reap system only deleted rows from `history_operation_participants`, `history_operations`, `history_transaction_participants`, `history_transactions`, `history_ledgers`, and `history_trades`. + ## 1.0.0 ### Before you upgrade diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 09463cd906..3a315fec6f 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -96,7 +96,9 @@ var dbReapCmd = &cobra.Command{ Short: "reaps (i.e. removes) any reapable history data", Long: "reap removes any historical data that is earlier than the configured retention cutoff", Run: func(cmd *cobra.Command, args []string) { - err := initApp().DeleteUnretainedHistory() + app := initApp() + app.UpdateLedgerState() + err := app.DeleteUnretainedHistory() if err != nil { log.Fatal(err) } diff --git a/services/horizon/internal/reap/main.go b/services/horizon/internal/reap/main.go index 91701c09a6..fc68d1c821 100644 --- a/services/horizon/internal/reap/main.go +++ b/services/horizon/internal/reap/main.go @@ -7,12 +7,13 @@ package reap import ( "time" + "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/db" ) // System represents the history reaping subsystem of horizon. type System struct { - HorizonDB *db.Session + HistoryQ *history.Q RetentionCount uint nextRun time.Time @@ -20,9 +21,9 @@ type System struct { // New initializes the reaper, causing it to begin polling the stellar-core // database for now ledgers and ingesting data into the horizon database. -func New(retention uint, horizon *db.Session) *System { +func New(retention uint, dbSession *db.Session) *System { r := &System{ - HorizonDB: horizon, + HistoryQ: &history.Q{dbSession}, RetentionCount: retention, } diff --git a/services/horizon/internal/reap/system.go b/services/horizon/internal/reap/system.go index 79510c6ef2..beaaf6e231 100644 --- a/services/horizon/internal/reap/system.go +++ b/services/horizon/internal/reap/system.go @@ -66,30 +66,12 @@ func (r *System) runOnce() { func (r *System) clearBefore(seq int32) error { log.WithField("new_elder", seq).Info("reaper: clearing") - clear := r.HorizonDB.DeleteRange - end := toid.New(seq, 0, 0).ToInt64() - - err := clear(0, end, "history_effects", "history_operation_id") - if err != nil { - return err - } - err = clear(0, end, "history_operation_participants", "history_operation_id") - if err != nil { - return err - } - err = clear(0, end, "history_operations", "id") + start, end, err := toid.LedgerRangeInclusive(1, seq-1) if err != nil { return err } - err = clear(0, end, "history_transaction_participants", "history_transaction_id") - if err != nil { - return err - } - err = clear(0, end, "history_transactions", "id") - if err != nil { - return err - } - err = clear(0, end, "history_ledgers", "id") + + err = r.HistoryQ.DeleteRangeAll(start, end) if err != nil { return err } From 18765c295b1f85c9e09dc32357f73de9407ece7c Mon Sep 17 00:00:00 2001 From: tamirms Date: Fri, 28 Feb 2020 16:52:23 +0100 Subject: [PATCH 2/3] Update services/horizon/CHANGELOG.md Co-Authored-By: Bartek Nowotarski --- services/horizon/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index e5eaba840e..281c485399 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -6,7 +6,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ## 1.1.0 * Fix `horizon db reap` bug which caused the command to exit without deleting any history table rows. -* The horizon reap system now also deletes rows from `history_effects`. Previously, the reap system only deleted rows from `history_operation_participants`, `history_operations`, `history_transaction_participants`, `history_transactions`, `history_ledgers`, and `history_trades`. +* The horizon reap system now also deletes rows from `history_trades`. Previously, the reap system only deleted rows from `history_operation_participants`, `history_operations`, `history_transaction_participants`, `history_transactions`, `history_ledgers`, and `history_effects`. ## 1.0.0 From 612dde41cbc5b60e767cee390ee9b5d900ee8057 Mon Sep 17 00:00:00 2001 From: tamirms Date: Fri, 28 Feb 2020 16:54:08 +0100 Subject: [PATCH 3/3] List changes as Unreleased instead of under version 1.1.0 --- services/horizon/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 281c485399..c4d150c15b 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -3,7 +3,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). -## 1.1.0 +## Unreleased * Fix `horizon db reap` bug which caused the command to exit without deleting any history table rows. * The horizon reap system now also deletes rows from `history_trades`. Previously, the reap system only deleted rows from `history_operation_participants`, `history_operations`, `history_transaction_participants`, `history_transactions`, `history_ledgers`, and `history_effects`.