Skip to content

Commit

Permalink
go/worker/storage: Add initial sync from checkpoints
Browse files Browse the repository at this point in the history
  • Loading branch information
jberci committed Aug 20, 2020
1 parent 89225d6 commit dcd86c2
Show file tree
Hide file tree
Showing 6 changed files with 482 additions and 4 deletions.
3 changes: 3 additions & 0 deletions go/oasis-test-runner/oasis/fixture.go
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,8 @@ type StorageWorkerFixture struct { // nolint: maligned
AllowEarlyTermination bool `json:"allow_early_termination"`
AllowErrorTermination bool `json:"allow_error_termination"`

NoAutoStart bool `json:"no_auto_start,omitempty"`

DisableCertRotation bool `json:"disable_cert_rotation"`

LogWatcherHandlerFactories []log.WatcherHandlerFactory `json:"-"`
Expand Down Expand Up @@ -353,6 +355,7 @@ func (f *StorageWorkerFixture) Create(net *Network) (*Storage, error) {
NodeCfg: NodeCfg{
AllowEarlyTermination: f.AllowEarlyTermination,
AllowErrorTermination: f.AllowErrorTermination,
NoAutoStart: f.NoAutoStart,
LogWatcherHandlerFactories: f.LogWatcherHandlerFactories,
Consensus: f.Consensus,
},
Expand Down
15 changes: 14 additions & 1 deletion go/oasis-test-runner/oasis/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
tendermint "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/api"
"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/log"
roothash "github.com/oasisprotocol/oasis-core/go/roothash/api"
workerStorage "github.com/oasisprotocol/oasis-core/go/worker/storage/committee"
upgrade "github.com/oasisprotocol/oasis-core/go/upgrade/api"
)

Expand Down Expand Up @@ -75,8 +76,20 @@ func LogEventABCIPruneDelete() log.WatcherHandlerFactory {
return LogAssertEvent(abci.LogEventABCIPruneDelete, "expected ABCI pruning to be done")
}

// LogAssertRoothashRoothashReindexing returns a handler witch checks wether roothash reindexing was
// LogAssertRoothashRoothashReindexing returns a handler which checks whether roothash reindexing was
// run based on JSON log output.
func LogAssertRoothashRoothashReindexing() log.WatcherHandlerFactory {
return LogAssertEvent(roothash.LogEventHistoryReindexing, "roothash runtime reindexing not detected")
}

// LogAssertCheckpointSyncSuccess returns a handler which checks whether initial storage sync from
// a checkpoint was successful or not.
func LogAssertCheckpointSyncSuccess() log.WatcherHandlerFactory {
return LogAssertEvent(workerStorage.LogEventCheckpointSyncSuccess, "checkpoint sync did not succeed")
}

// LogAssertCheckpointSync returns a handler which checks whether initial storage sync from
// a checkpoint failed or not.
func LogAssertCheckpointSync() log.WatcherHandlerFactory {
return LogAssertNotEvent(workerStorage.LogEventCheckpointSyncFailed, "checkpoint sync failed")
}
1 change: 1 addition & 0 deletions go/oasis-test-runner/oasis/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ func (net *Network) NewStorage(cfg *StorageCfg) (*Storage, error) {
Name: storageName,
net: net,
dir: storageDir,
noAutoStart: cfg.NoAutoStart,
disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
logWatcherHandlerFactories: cfg.LogWatcherHandlerFactories,
consensus: cfg.Consensus,
Expand Down
23 changes: 23 additions & 0 deletions go/oasis-test-runner/scenario/e2e/runtime/storage_sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,18 @@ func (sc *storageSyncImpl) Fixture() (*oasis.NetworkFixture, error) {
Entity: 1,
IgnoreApplies: true,
})

// One more storage worker for later, so it can do an initial sync with the snapshots.
f.StorageWorkers = append(f.StorageWorkers, oasis.StorageWorkerFixture{
Backend: database.BackendNameBadgerDB,
Entity: 1,
NoAutoStart: true,
})
lateWorker := len(f.StorageWorkers)-1

// Add log assertion for the late worker.
f.StorageWorkers[lateWorker].LogWatcherHandlerFactories = append(f.StorageWorkers[lateWorker].LogWatcherHandlerFactories, oasis.LogAssertCheckpointSync())

return f, nil
}

Expand Down Expand Up @@ -161,5 +173,16 @@ func (sc *storageSyncImpl) Run(childEnv *env.Env) error {
return fmt.Errorf("incorrect number of valid checkpoints (expected: >=2 got: %d)", validCps)
}

// Now spin up the last storage worker and check if it syncs with a checkpoint.
lateWorker := sc.Net.StorageWorkers()[3]
err = lateWorker.Start()
if err != nil {
return fmt.Errorf("can't start last storage worker: %w", err)
}
err = lateWorker.WaitReady(ctx)
if err != nil {
return fmt.Errorf("error waiting for last storage worker to become ready: %w", err)
}

return nil
}
Loading

0 comments on commit dcd86c2

Please sign in to comment.