From c964f68ef143c6d23b7f1eaec2599f99972f690d Mon Sep 17 00:00:00 2001 From: Brennan Lamey Date: Wed, 26 Jun 2024 17:16:42 -0500 Subject: [PATCH 1/2] fix crash on invalid snapshot --- internal/statesync/store.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/statesync/store.go b/internal/statesync/store.go index 837803829..118d40327 100644 --- a/internal/statesync/store.go +++ b/internal/statesync/store.go @@ -245,6 +245,7 @@ func (s *SnapshotStore) loadSnapshots() error { heightInt, err := strconv.ParseUint(height, 10, 64) if err != nil { s.log.Debug("invalid snapshot height, ignoring the snapshot", log.String("height", height)) + continue } // Load snapshot header @@ -252,6 +253,7 @@ func (s *SnapshotStore) loadSnapshots() error { header, err := loadSnapshot(headerFile) if err != nil { s.log.Debug("Invalid snapshot header file, ignoring the snapshot", log.String("height", height), log.String("Error", err.Error())) + continue } // Ensure that the chunk files exist @@ -259,6 +261,7 @@ func (s *SnapshotStore) loadSnapshots() error { chunkFile := snapshotChunkFile(s.cfg.SnapshotDir, heightInt, DefaultSnapshotFormat, i) if _, err := os.Open(chunkFile); err != nil { // chunk file doesn't exist s.log.Debug("Invalid snapshot chunk file, ignoring the snapshot", log.String("chunk-file", chunkFile)) + continue } } From c7cc4d6210cb2209f9db4d8df99dbcc1b5185918 Mon Sep 17 00:00:00 2001 From: Jonathan Chappelow Date: Wed, 26 Jun 2024 18:20:28 -0500 Subject: [PATCH 2/2] smarter snapshot chunk loading --- internal/statesync/store.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/internal/statesync/store.go b/internal/statesync/store.go index 118d40327..04c3ea7b5 100644 --- a/internal/statesync/store.go +++ b/internal/statesync/store.go @@ -2,6 +2,7 @@ package statesync import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -205,16 +206,17 @@ func (s *SnapshotStore) LoadSnapshotChunk(height uint64, format uint32, chunkIdx return nil, fmt.Errorf("chunk %d does not exist in snapshot at height %d", chunkIdx, height) } - chunkFile := snapshotChunkFile(s.cfg.SnapshotDir, height, format, chunkIdx) - if _, err := os.Open(chunkFile); err != nil { - return nil, fmt.Errorf("chunk %d does not exist in snapshot at height %d", chunkIdx, height) - } - // Read the chunk file + chunkFile := snapshotChunkFile(s.cfg.SnapshotDir, height, format, chunkIdx) bts, err := os.ReadFile(chunkFile) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("chunk %d does not exist in snapshot at height %d", chunkIdx, height) + } return nil, fmt.Errorf("failed to read chunk %d at height %d: %w", chunkIdx, height, err) } + // TODO: cache chunk bytes by file name (with last used vacuum timer) so + // many peers don't force us to load the same chunk into memory many times. return bts, nil } @@ -238,13 +240,13 @@ func (s *SnapshotStore) loadSnapshots() error { fileName := file.Name() // format: block- names := strings.Split(fileName, "-") if len(names) != 2 { - s.log.Debug("invalid snapshot directory name, ignoring the snapshot", log.String("dir", fileName)) + s.log.Warn("invalid snapshot directory name, ignoring the snapshot", log.String("dir", fileName)) continue } height := names[1] heightInt, err := strconv.ParseUint(height, 10, 64) if err != nil { - s.log.Debug("invalid snapshot height, ignoring the snapshot", log.String("height", height)) + s.log.Warn("invalid snapshot height, ignoring the snapshot", log.String("height", height)) continue } @@ -252,15 +254,17 @@ func (s *SnapshotStore) loadSnapshots() error { headerFile := snapshotHeaderFile(s.cfg.SnapshotDir, heightInt, DefaultSnapshotFormat) header, err := loadSnapshot(headerFile) if err != nil { - s.log.Debug("Invalid snapshot header file, ignoring the snapshot", log.String("height", height), log.String("Error", err.Error())) + s.log.Warn("Invalid snapshot header file, ignoring the snapshot", + log.String("height", height), log.Error(err)) continue } // Ensure that the chunk files exist for i := uint32(0); i < header.ChunkCount; i++ { chunkFile := snapshotChunkFile(s.cfg.SnapshotDir, heightInt, DefaultSnapshotFormat, i) - if _, err := os.Open(chunkFile); err != nil { // chunk file doesn't exist - s.log.Debug("Invalid snapshot chunk file, ignoring the snapshot", log.String("chunk-file", chunkFile)) + if _, err := os.Stat(chunkFile); err != nil { // chunk file doesn't exist + s.log.Warn("Invalid snapshot chunk file, ignoring the snapshot", + log.String("chunk_file", chunkFile), log.Error(err)) continue } }