From 6dac40b4f873e41865d3a67604804ca4884d41fb Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 13:45:03 -0700 Subject: [PATCH 01/10] Add tool to replay changelog --- .../cmd/seidb/operations/replay_changelog.go | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tools/cmd/seidb/operations/replay_changelog.go diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go new file mode 100644 index 0000000..3672ef5 --- /dev/null +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -0,0 +1,57 @@ +package operations + +import ( + "fmt" + "github.com/sei-protocol/sei-db/proto" + "github.com/sei-protocol/sei-db/stream/changelog" + "path/filepath" + + "github.com/sei-protocol/sei-db/common/logger" + "github.com/spf13/cobra" +) + +func ReplayChangelogCmd() *cobra.Command { + dumpDbCmd := &cobra.Command{ + Use: "replay-changelog", + Short: "Scan the changelog to replay and recover pebbledb data", + Run: executeReplayChangelog, + } + + dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") + dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset") + dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 1, "To offset") + + return dumpDbCmd +} + +func executeReplayChangelog(cmd *cobra.Command, _ []string) { + dbDir, _ := cmd.Flags().GetString("db-dir") + start, _ := cmd.Flags().GetInt64("start-offset") + end, _ := cmd.Flags().GetInt64("end-offset") + if dbDir == "" { + panic("Must provide database dir") + } + if start > end || start < 0 { + panic("Must provide a valid start/end offset") + } + logDir := filepath.Join(dbDir, "changelog") + stream, err := changelog.NewStream(logger.NewNopLogger(), logDir, changelog.Config{}) + if err != nil { + panic(err) + } + err = stream.Replay(uint64(start), uint64(end), processChangelogEntry) + if err != nil { + panic(err) + } +} + +func processChangelogEntry(index uint64, entry proto.ChangelogEntry) error { + fmt.Printf("Index: %d, version: %d\n", index, entry.Version) + for _, store := range entry.Changesets { + storeName := store.Name + for _, kv := range store.Changeset.Pairs { + fmt.Printf("store: %s: key:%s\n", storeName, string(kv.Key)) + } + } + return nil +} From 648ce358e488520dd32fcc4c13bd63c8ff216365 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 13:47:34 -0700 Subject: [PATCH 02/10] Fix command --- tools/cmd/seidb/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/cmd/seidb/main.go b/tools/cmd/seidb/main.go index 360ad2a..cef02f3 100644 --- a/tools/cmd/seidb/main.go +++ b/tools/cmd/seidb/main.go @@ -24,7 +24,8 @@ func main() { operations.DumpDbCmd(), operations.PruneCmd(), operations.DumpIAVLCmd(), - operations.StateSizeCmd()) + operations.StateSizeCmd(), + operations.ReplayChangelogCmd()) if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) From 31fb39101139f8f15cbca7357456ad1566c94b4b Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 13:51:42 -0700 Subject: [PATCH 03/10] Fix --- tools/cmd/seidb/operations/replay_changelog.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 3672ef5..da5048c 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -46,11 +46,11 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { } func processChangelogEntry(index uint64, entry proto.ChangelogEntry) error { - fmt.Printf("Index: %d, version: %d\n", index, entry.Version) + fmt.Printf("Offset: %d, Height: %d\n", index, entry.Version) for _, store := range entry.Changesets { storeName := store.Name for _, kv := range store.Changeset.Pairs { - fmt.Printf("store: %s: key:%s\n", storeName, string(kv.Key)) + fmt.Printf("store: %s, key: %X\n", storeName, kv.Key) } } return nil From 48c979628a12f01a74758c4c484c62ab4edfb2b0 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 14:04:24 -0700 Subject: [PATCH 04/10] Add height --- .../cmd/seidb/operations/replay_changelog.go | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index da5048c..621ad2c 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -18,16 +18,16 @@ func ReplayChangelogCmd() *cobra.Command { } dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") - dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset") - dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 1, "To offset") + dumpDbCmd.PersistentFlags().Int64P("start-height", "s", 0, "From offset") + dumpDbCmd.PersistentFlags().Int64P("end-height", "e", 1, "To offset") return dumpDbCmd } func executeReplayChangelog(cmd *cobra.Command, _ []string) { dbDir, _ := cmd.Flags().GetString("db-dir") - start, _ := cmd.Flags().GetInt64("start-offset") - end, _ := cmd.Flags().GetInt64("end-offset") + start, _ := cmd.Flags().GetInt64("start-height") + end, _ := cmd.Flags().GetInt64("end-height") if dbDir == "" { panic("Must provide database dir") } @@ -39,7 +39,18 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { if err != nil { panic(err) } - err = stream.Replay(uint64(start), uint64(end), processChangelogEntry) + // Read first entry to compute the difference between offset and height + firstOffset, err := stream.FirstOffset() + if err != nil { + panic(err) + } + firstEntry, err := stream.ReadAt(firstOffset) + if err != nil { + panic(err) + } + gap := firstEntry.Version - int64(firstOffset) + + err = stream.Replay(uint64(start-gap), uint64(end-gap), processChangelogEntry) if err != nil { panic(err) } From 43c2cbcbd3e45521ab139b6db0f090d8b763ccb7 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 14:06:33 -0700 Subject: [PATCH 05/10] Add log --- tools/cmd/seidb/operations/replay_changelog.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 621ad2c..2d73535 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -26,12 +26,12 @@ func ReplayChangelogCmd() *cobra.Command { func executeReplayChangelog(cmd *cobra.Command, _ []string) { dbDir, _ := cmd.Flags().GetString("db-dir") - start, _ := cmd.Flags().GetInt64("start-height") - end, _ := cmd.Flags().GetInt64("end-height") + startHeight, _ := cmd.Flags().GetInt64("start-height") + endHeight, _ := cmd.Flags().GetInt64("end-height") if dbDir == "" { panic("Must provide database dir") } - if start > end || start < 0 { + if startHeight > endHeight || startHeight < 0 { panic("Must provide a valid start/end offset") } logDir := filepath.Join(dbDir, "changelog") @@ -49,8 +49,10 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { panic(err) } gap := firstEntry.Version - int64(firstOffset) - - err = stream.Replay(uint64(start-gap), uint64(end-gap), processChangelogEntry) + startOffset := uint64(startHeight - gap) + endOffset := uint64(endHeight - gap) + fmt.Printf("Replaying changelog from offset %d to %d\n", startOffset, endOffset) + err = stream.Replay(startOffset, endOffset, processChangelogEntry) if err != nil { panic(err) } From e9153c58f287618dbfe1ae35be94b98d0009fd58 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 14:21:51 -0700 Subject: [PATCH 06/10] Add repair mode for changelog --- .../cmd/seidb/operations/replay_changelog.go | 55 ++++++++++++------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 2d73535..3a76786 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -2,14 +2,19 @@ package operations import ( "fmt" - "github.com/sei-protocol/sei-db/proto" - "github.com/sei-protocol/sei-db/stream/changelog" "path/filepath" "github.com/sei-protocol/sei-db/common/logger" + "github.com/sei-protocol/sei-db/config" + "github.com/sei-protocol/sei-db/proto" + "github.com/sei-protocol/sei-db/ss" + "github.com/sei-protocol/sei-db/ss/types" + "github.com/sei-protocol/sei-db/stream/changelog" "github.com/spf13/cobra" ) +var ssStore types.StateStore + func ReplayChangelogCmd() *cobra.Command { dumpDbCmd := &cobra.Command{ Use: "replay-changelog", @@ -18,20 +23,22 @@ func ReplayChangelogCmd() *cobra.Command { } dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") - dumpDbCmd.PersistentFlags().Int64P("start-height", "s", 0, "From offset") - dumpDbCmd.PersistentFlags().Int64P("end-height", "e", 1, "To offset") + dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset") + dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 1, "End offset") + dumpDbCmd.PersistentFlags().BoolP("apply", "a", false, "Whether to re-apply the changelog or not") return dumpDbCmd } func executeReplayChangelog(cmd *cobra.Command, _ []string) { dbDir, _ := cmd.Flags().GetString("db-dir") - startHeight, _ := cmd.Flags().GetInt64("start-height") - endHeight, _ := cmd.Flags().GetInt64("end-height") + start, _ := cmd.Flags().GetInt64("start-offset") + end, _ := cmd.Flags().GetInt64("end-offset") if dbDir == "" { panic("Must provide database dir") } - if startHeight > endHeight || startHeight < 0 { + + if start > end || start < 0 { panic("Must provide a valid start/end offset") } logDir := filepath.Join(dbDir, "changelog") @@ -39,32 +46,38 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { if err != nil { panic(err) } - // Read first entry to compute the difference between offset and height - firstOffset, err := stream.FirstOffset() - if err != nil { - panic(err) - } - firstEntry, err := stream.ReadAt(firstOffset) + + // open the database + ssConfig := config.DefaultStateStoreConfig() + ssConfig.KeepRecent = 0 + ssConfig.DBDirectory = dbDir + ssStore, err = ss.NewStateStore(logger.NewNopLogger(), dbDir, ssConfig) if err != nil { panic(err) } - gap := firstEntry.Version - int64(firstOffset) - startOffset := uint64(startHeight - gap) - endOffset := uint64(endHeight - gap) - fmt.Printf("Replaying changelog from offset %d to %d\n", startOffset, endOffset) - err = stream.Replay(startOffset, endOffset, processChangelogEntry) + + // replay the changelog + err = stream.Replay(uint64(start), uint64(end), processChangelogEntry) if err != nil { panic(err) } + + // close the database + ssStore.Close() + } func processChangelogEntry(index uint64, entry proto.ChangelogEntry) error { fmt.Printf("Offset: %d, Height: %d\n", index, entry.Version) - for _, store := range entry.Changesets { - storeName := store.Name - for _, kv := range store.Changeset.Pairs { + for _, changeset := range entry.Changesets { + storeName := changeset.Name + for _, kv := range changeset.Changeset.Pairs { fmt.Printf("store: %s, key: %X\n", storeName, kv.Key) } + err := ssStore.ApplyChangeset(entry.Version, changeset) + if err != nil { + return err + } } return nil } From b8ca3d528447c7103406236ce6eef8feaafbe566 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 14:27:17 -0700 Subject: [PATCH 07/10] Add dry run option --- .../cmd/seidb/operations/replay_changelog.go | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 3a76786..10550c3 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -25,7 +25,7 @@ func ReplayChangelogCmd() *cobra.Command { dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset") dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 1, "End offset") - dumpDbCmd.PersistentFlags().BoolP("apply", "a", false, "Whether to re-apply the changelog or not") + dumpDbCmd.PersistentFlags().Bool("dry-run", true, "Whether to dry run or re-apply the changelog to DB") return dumpDbCmd } @@ -34,6 +34,7 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { dbDir, _ := cmd.Flags().GetString("db-dir") start, _ := cmd.Flags().GetInt64("start-offset") end, _ := cmd.Flags().GetInt64("end-offset") + dryRun, _ := cmd.Flags().GetBool("dry-run") if dbDir == "" { panic("Must provide database dir") } @@ -47,13 +48,15 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { panic(err) } - // open the database - ssConfig := config.DefaultStateStoreConfig() - ssConfig.KeepRecent = 0 - ssConfig.DBDirectory = dbDir - ssStore, err = ss.NewStateStore(logger.NewNopLogger(), dbDir, ssConfig) - if err != nil { - panic(err) + // open the database if apply is true + if !dryRun { + ssConfig := config.DefaultStateStoreConfig() + ssConfig.KeepRecent = 0 + ssConfig.DBDirectory = dbDir + ssStore, err = ss.NewStateStore(logger.NewNopLogger(), dbDir, ssConfig) + if err != nil { + panic(err) + } } // replay the changelog @@ -74,9 +77,11 @@ func processChangelogEntry(index uint64, entry proto.ChangelogEntry) error { for _, kv := range changeset.Changeset.Pairs { fmt.Printf("store: %s, key: %X\n", storeName, kv.Key) } - err := ssStore.ApplyChangeset(entry.Version, changeset) - if err != nil { - return err + if ssStore != nil { + err := ssStore.ApplyChangeset(entry.Version, changeset) + if err != nil { + return err + } } } return nil From 6e0b75cec4ec981978d40152cc94bcaf07effb75 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 14:29:36 -0700 Subject: [PATCH 08/10] Fix --- tools/cmd/seidb/operations/replay_changelog.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 10550c3..7b9b275 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -66,7 +66,9 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { } // close the database - ssStore.Close() + if ssStore != nil { + ssStore.Close() + } } From c350de55256fd8f80f954727000a9966701917a2 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 14:33:43 -0700 Subject: [PATCH 09/10] Add no dry run --- tools/cmd/seidb/operations/replay_changelog.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 7b9b275..61930bc 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -25,7 +25,7 @@ func ReplayChangelogCmd() *cobra.Command { dumpDbCmd.PersistentFlags().StringP("db-dir", "d", "", "Database Directory") dumpDbCmd.PersistentFlags().Int64P("start-offset", "s", 0, "From offset") dumpDbCmd.PersistentFlags().Int64P("end-offset", "e", 1, "End offset") - dumpDbCmd.PersistentFlags().Bool("dry-run", true, "Whether to dry run or re-apply the changelog to DB") + dumpDbCmd.PersistentFlags().Bool("no-dry-run", false, "Whether to dry run or re-apply the changelog to DB") return dumpDbCmd } @@ -34,7 +34,7 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { dbDir, _ := cmd.Flags().GetString("db-dir") start, _ := cmd.Flags().GetInt64("start-offset") end, _ := cmd.Flags().GetInt64("end-offset") - dryRun, _ := cmd.Flags().GetBool("dry-run") + noDryRun, _ := cmd.Flags().GetBool("no-dry-run") if dbDir == "" { panic("Must provide database dir") } @@ -49,7 +49,7 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { } // open the database if apply is true - if !dryRun { + if noDryRun { ssConfig := config.DefaultStateStoreConfig() ssConfig.KeepRecent = 0 ssConfig.DBDirectory = dbDir @@ -80,6 +80,7 @@ func processChangelogEntry(index uint64, entry proto.ChangelogEntry) error { fmt.Printf("store: %s, key: %X\n", storeName, kv.Key) } if ssStore != nil { + fmt.Printf("Re-applied changeset for height %d\n", entry.Version) err := ssStore.ApplyChangeset(entry.Version, changeset) if err != nil { return err From ba81c48a235f748556e417ac2222bfdb3d537b70 Mon Sep 17 00:00:00 2001 From: yzang2019 Date: Sat, 10 Aug 2024 14:40:06 -0700 Subject: [PATCH 10/10] Fix comment --- tools/cmd/seidb/operations/replay_changelog.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cmd/seidb/operations/replay_changelog.go b/tools/cmd/seidb/operations/replay_changelog.go index 61930bc..0db0fde 100644 --- a/tools/cmd/seidb/operations/replay_changelog.go +++ b/tools/cmd/seidb/operations/replay_changelog.go @@ -48,7 +48,7 @@ func executeReplayChangelog(cmd *cobra.Command, _ []string) { panic(err) } - // open the database if apply is true + // open the database if this is not a dry run if noDryRun { ssConfig := config.DefaultStateStoreConfig() ssConfig.KeepRecent = 0