From 504d0e8864b0d1f776b282da29c364c92313a148 Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Wed, 20 Oct 2021 23:13:45 +0200 Subject: [PATCH 1/4] ingest/ledgerbackend: Make sure Stellar-Core is not started before previous instance termination --- ingest/ledgerbackend/captive_core_backend.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go index 5f4e50cbd0..435050ebe6 100644 --- a/ingest/ledgerbackend/captive_core_backend.go +++ b/ingest/ledgerbackend/captive_core_backend.go @@ -347,6 +347,12 @@ func (c *CaptiveStellarCore) startPreparingRange(ctx context.Context, ledgerRang if err := c.stellarCoreRunner.close(); err != nil { return false, errors.Wrap(err, "error closing existing session") } + + // Make sure Stellar-Core is terminated before starting a new instance. + processExited, _ := c.stellarCoreRunner.getProcessExitError() + if !processExited { + return false, errors.New("the previous Stellar-Core instance is still running") + } } var err error From 8aa2a522446d1f41a22ed0e73446ad0103b15f4c Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Wed, 20 Oct 2021 23:29:03 +0200 Subject: [PATCH 2/4] Add test --- .../captive_core_backend_test.go | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go index 67c390c0d5..5d9d9c4477 100644 --- a/ingest/ledgerbackend/captive_core_backend_test.go +++ b/ingest/ledgerbackend/captive_core_backend_test.go @@ -266,6 +266,50 @@ func TestCaptivePrepareRangeTerminated(t *testing.T) { mockArchive.AssertExpectations(t) } +func TestCaptivePrepareRangeCloseNotFullyTerminated(t *testing.T) { + metaChan := make(chan metaResult, 100) + for i := 64; i <= 100; i++ { + meta := buildLedgerCloseMeta(testLedgerHeader{sequence: uint32(i)}) + metaChan <- metaResult{ + LedgerCloseMeta: &meta, + } + } + + ctx, cancel := context.WithCancel(context.Background()) + mockRunner := &stellarCoreRunnerMock{} + mockRunner.On("catchup", uint32(100), uint32(200)).Return(nil).Once() + mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan)) + mockRunner.On("context").Return(ctx) + + mockArchive := &historyarchive.MockArchive{} + mockArchive. + On("GetRootHAS"). + Return(historyarchive.HistoryArchiveState{ + CurrentLedger: uint32(200), + }, nil) + + captiveBackend := CaptiveStellarCore{ + archive: mockArchive, + stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) { + return mockRunner, nil + }, + checkpointManager: historyarchive.NewCheckpointManager(64), + } + + err := captiveBackend.PrepareRange(ctx, BoundedRange(100, 200)) + assert.NoError(t, err) + + cancel() + mockRunner.On("close").Return(nil) + mockRunner.On("getProcessExitError").Return(false, nil).Once() + + err = captiveBackend.PrepareRange(ctx, BoundedRange(100, 200)) + assert.EqualError(t, err, "error starting prepare range: the previous Stellar-Core instance is still running") + + mockRunner.AssertExpectations(t) + mockArchive.AssertExpectations(t) +} + func TestCaptivePrepareRange_ErrClosingSession(t *testing.T) { ctx := context.Background() mockRunner := &stellarCoreRunnerMock{} From aade63cb847b98c69b5284744426ac688b7a33a8 Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Wed, 20 Oct 2021 23:31:32 +0200 Subject: [PATCH 3/4] Add comment --- ingest/ledgerbackend/captive_core_backend_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go index 5d9d9c4477..4645144cdd 100644 --- a/ingest/ledgerbackend/captive_core_backend_test.go +++ b/ingest/ledgerbackend/captive_core_backend_test.go @@ -299,6 +299,7 @@ func TestCaptivePrepareRangeCloseNotFullyTerminated(t *testing.T) { err := captiveBackend.PrepareRange(ctx, BoundedRange(100, 200)) assert.NoError(t, err) + // Simulates a long (but graceful) shutdown... cancel() mockRunner.On("close").Return(nil) mockRunner.On("getProcessExitError").Return(false, nil).Once() From e1517589c8c8245636a47a87dbecd093aaec5f25 Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Thu, 21 Oct 2021 00:53:23 +0200 Subject: [PATCH 4/4] Fix --- ingest/ledgerbackend/stellar_core_runner.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go index cf054cb4d1..2addb75038 100644 --- a/ingest/ledgerbackend/stellar_core_runner.go +++ b/ingest/ledgerbackend/stellar_core_runner.go @@ -461,6 +461,12 @@ func (r *stellarCoreRunner) close() error { return nil } + if !started { + // Update processExited if handleExit that updates it not even started + // (error before command run). + r.processExited = true + } + r.cancel() r.lock.Unlock()