diff --git a/ingest/CHANGELOG.md b/ingest/CHANGELOG.md index 03c41c6dff..f310258566 100644 --- a/ingest/CHANGELOG.md +++ b/ingest/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. This projec ## Unreleased +* Let filewatcher use binary hash instead of timestap to detect core version update. [4050](https://github.com/stellar/go/pull/4050) + ### New Features * **Performance improvement**: the Captive Core backend now reuses bucket files whenever it finds existing ones in the corresponding `--captive-core-storage-path` (introduced in [v2.0](#v2.0.0)) rather than generating a one-time temporary sub-directory ([#3670](https://github.com/stellar/go/pull/3670)). Note that taking advantage of this feature requires [Stellar-Core v17.1.0](https://github.com/stellar/stellar-core/releases/tag/v17.1.0) or later. diff --git a/ingest/ledgerbackend/file_watcher.go b/ingest/ledgerbackend/file_watcher.go index af68596e3d..7b468d9378 100644 --- a/ingest/ledgerbackend/file_watcher.go +++ b/ingest/ledgerbackend/file_watcher.go @@ -1,37 +1,60 @@ package ledgerbackend import ( + "bytes" + "crypto/sha1" + "io" "os" "sync" "time" - "github.com/pkg/errors" - + "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" ) +type hash []byte + +func (h hash) Equals(other hash) bool { + return bytes.Equal(h, other) +} + type fileWatcher struct { - pathToFile string - duration time.Duration - onChange func() - exit <-chan struct{} - log *log.Entry - stat func(string) (os.FileInfo, error) - lastModTime time.Time + pathToFile string + duration time.Duration + onChange func() + exit <-chan struct{} + log *log.Entry + hashFile func(string) (hash, error) + lastHash hash +} + +func hashFile(filename string) (hash, error) { + f, err := os.Open(filename) + if err != nil { + return hash{}, errors.Wrapf(err, "unable to open %v", f) + } + defer f.Close() + + h := sha1.New() + if _, err := io.Copy(h, f); err != nil { + return hash{}, errors.Wrapf(err, "unable to copy %v into buffer", f) + } + + return h.Sum(nil), nil } func newFileWatcher(runner *stellarCoreRunner) (*fileWatcher, error) { - return newFileWatcherWithOptions(runner, os.Stat, 10*time.Second) + return newFileWatcherWithOptions(runner, hashFile, 10*time.Second) } func newFileWatcherWithOptions( runner *stellarCoreRunner, - stat func(string) (os.FileInfo, error), + hashFile func(string) (hash, error), tickerDuration time.Duration, ) (*fileWatcher, error) { - info, err := stat(runner.executablePath) + hashResult, err := hashFile(runner.executablePath) if err != nil { - return nil, errors.Wrap(err, "could not stat captive core binary") + return nil, errors.Wrap(err, "could not hash captive core binary") } once := &sync.Once{} @@ -46,10 +69,10 @@ func newFileWatcherWithOptions( } }) }, - exit: runner.ctx.Done(), - log: runner.log, - stat: stat, - lastModTime: info.ModTime(), + exit: runner.ctx.Done(), + log: runner.log, + hashFile: hashFile, + lastHash: hashResult, }, nil } @@ -70,18 +93,18 @@ func (f *fileWatcher) loop() { } func (f *fileWatcher) fileChanged() bool { - info, err := f.stat(f.pathToFile) + hashResult, err := f.hashFile(f.pathToFile) if err != nil { - f.log.Warnf("could not stat %s: %v", f.pathToFile, err) + f.log.Warnf("could not hash contents of %s: %v", f.pathToFile, err) return false } - if modTime := info.ModTime(); !f.lastModTime.Equal(modTime) { + if !f.lastHash.Equals(hashResult) { f.log.Infof( - "detected update to %s. previous file timestamp was %v current timestamp is %v", + "detected update to %s. previous file hash was %v current hash is %v", f.pathToFile, - f.lastModTime, - modTime, + f.lastHash, + hashResult, ) return true } diff --git a/ingest/ledgerbackend/file_watcher_test.go b/ingest/ledgerbackend/file_watcher_test.go index a2720c6342..e46606d2e8 100644 --- a/ingest/ledgerbackend/file_watcher_test.go +++ b/ingest/ledgerbackend/file_watcher_test.go @@ -3,7 +3,6 @@ package ledgerbackend import ( "context" "fmt" - "os" "sync" "testing" "time" @@ -13,67 +12,39 @@ import ( "github.com/stretchr/testify/assert" ) -type mockFile struct { - modTime time.Time -} - -func (mockFile) Name() string { - return "" -} - -func (mockFile) Size() int64 { - return 0 -} - -func (mockFile) Mode() os.FileMode { - return 0 -} - -func (mockFile) IsDir() bool { - return false -} - -func (mockFile) Sys() interface{} { - return nil -} -func (m mockFile) ModTime() time.Time { - return m.modTime -} - -type mockStat struct { +type mockHash struct { sync.Mutex t *testing.T expectedPath string - modTime time.Time + hashResult hash err error callCount int } -func (m *mockStat) setResponse(modTime time.Time, err error) { +func (m *mockHash) setResponse(hashResult hash, err error) { m.Lock() defer m.Unlock() - m.modTime = modTime + m.hashResult = hashResult m.err = err } -func (m *mockStat) getCallCount() int { +func (m *mockHash) getCallCount() int { m.Lock() defer m.Unlock() return m.callCount } -func (m *mockStat) stat(fp string) (os.FileInfo, error) { +func (m *mockHash) hashFile(fp string) (hash, error) { m.Lock() defer m.Unlock() m.callCount++ assert.Equal(m.t, m.expectedPath, fp) - //defer m.onCall(m) - return mockFile{m.modTime}, m.err + return m.hashResult, m.err } -func createFWFixtures(t *testing.T) (*mockStat, *stellarCoreRunner, *fileWatcher) { - ms := &mockStat{ - modTime: time.Now(), +func createFWFixtures(t *testing.T) (*mockHash, *stellarCoreRunner, *fileWatcher) { + ms := &mockHash{ + hashResult: hash{}, expectedPath: "/some/path", t: t, } @@ -90,7 +61,7 @@ func createFWFixtures(t *testing.T) (*mockStat, *stellarCoreRunner, *fileWatcher }, stellarCoreRunnerModeOffline) assert.NoError(t, err) - fw, err := newFileWatcherWithOptions(runner, ms.stat, time.Millisecond) + fw, err := newFileWatcherWithOptions(runner, ms.hashFile, time.Millisecond) assert.NoError(t, err) assert.Equal(t, 1, ms.getCallCount()) @@ -98,12 +69,12 @@ func createFWFixtures(t *testing.T) (*mockStat, *stellarCoreRunner, *fileWatcher } func TestNewFileWatcherError(t *testing.T) { - ms := &mockStat{ - modTime: time.Now(), + ms := &mockHash{ + hashResult: hash{}, expectedPath: "/some/path", t: t, } - ms.setResponse(time.Time{}, fmt.Errorf("test error")) + ms.setResponse(hash{}, fmt.Errorf("test error")) captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{}) assert.NoError(t, err) @@ -117,29 +88,27 @@ func TestNewFileWatcherError(t *testing.T) { }, stellarCoreRunnerModeOffline) assert.NoError(t, err) - _, err = newFileWatcherWithOptions(runner, ms.stat, time.Millisecond) - assert.EqualError(t, err, "could not stat captive core binary: test error") + _, err = newFileWatcherWithOptions(runner, ms.hashFile, time.Millisecond) + assert.EqualError(t, err, "could not hash captive core binary: test error") assert.Equal(t, 1, ms.getCallCount()) } func TestFileChanged(t *testing.T) { ms, _, fw := createFWFixtures(t) - modTime := ms.modTime - assert.False(t, fw.fileChanged()) assert.False(t, fw.fileChanged()) assert.Equal(t, 3, ms.getCallCount()) - ms.setResponse(time.Time{}, fmt.Errorf("test error")) + ms.setResponse(hash{}, fmt.Errorf("test error")) assert.False(t, fw.fileChanged()) assert.Equal(t, 4, ms.getCallCount()) - ms.setResponse(modTime, nil) + ms.setResponse(ms.hashResult, nil) assert.False(t, fw.fileChanged()) assert.Equal(t, 5, ms.getCallCount()) - ms.setResponse(time.Now().Add(time.Hour), nil) + ms.setResponse(hash{1}, nil) assert.True(t, fw.fileChanged()) assert.Equal(t, 6, ms.getCallCount()) } @@ -161,7 +130,7 @@ func TestCloseRunnerDuringFileWatcherLoop(t *testing.T) { close(done) }() - // fw.loop will repeatedly check if the file has changed by calling stat. + // fw.loop will repeatedly check if the file has changed by calling hash. // This test ensures that closing the runner will exit fw.loop so that the goroutine is not leaked. closedRunner := false @@ -187,7 +156,7 @@ func TestFileChangesTriggerRunnerClose(t *testing.T) { close(done) }() - // fw.loop will repeatedly check if the file has changed by calling stat. + // fw.loop will repeatedly check if the file has changed by calling hash // This test ensures that modifying the file will trigger the closing of the runner. modifiedFile := false for { @@ -199,7 +168,7 @@ func TestFileChangesTriggerRunnerClose(t *testing.T) { return default: if ms.getCallCount() > 20 { - ms.setResponse(time.Now().Add(time.Hour), nil) + ms.setResponse(hash{1}, nil) modifiedFile = true } }