diff --git a/registry/exec/exec.go b/registry/exec/exec.go index bb0a65058..1c8b2251a 100644 --- a/registry/exec/exec.go +++ b/registry/exec/exec.go @@ -29,16 +29,18 @@ func NewRetryableExecutor[T any](opts ...OptsFn) RetryableExecutor[T] { func (r *retryableExecutor[T]) ExecWithFallback(execFn func() (T, error), fallbackFn func() error) (res T, err error) { if execFn == nil { - return res, errors.New("no exec function provided") + return res, ErrMissingExecFn } if fallbackFn == nil { - return res, errors.New("no fallback function provided") + return res, ErrMissingFallbackFn } execErr := &Error{} - for i := uint(0); i < r.opts.maxCount; i++ { + retryCount := uint(0) + + for { res, err = execFn() if err == nil { @@ -47,33 +49,42 @@ func (r *retryableExecutor[T]) ExecWithFallback(execFn func() (T, error), fallba execErr.AddErr(fmt.Errorf("exec function error: %w", err)) + if retryCount == r.opts.maxRetryCount { + return res, execErr + } + if err = fallbackFn(); err != nil && !r.opts.retryOnFallbackError { execErr.AddErr(fmt.Errorf("fallback function error: %w", err)) return res, execErr } + retryCount++ + time.Sleep(r.opts.errTimeout) } - - return res, execErr } +var ( + ErrMissingExecFn = errors.New("no exec function provided") + ErrMissingFallbackFn = errors.New("no fallback function provided") +) + const ( - defaultExecMaxCount = 3 + defaultMaxRetryCount = 3 defaultErrTimeout = 0 * time.Second defaultRetryOnFallbackError = true ) type Opts struct { - maxCount uint + maxRetryCount uint errTimeout time.Duration retryOnFallbackError bool } func NewDefaultExecOpts() *Opts { return &Opts{ - maxCount: defaultExecMaxCount, + maxRetryCount: defaultMaxRetryCount, errTimeout: defaultErrTimeout, retryOnFallbackError: defaultRetryOnFallbackError, } @@ -81,9 +92,13 @@ func NewDefaultExecOpts() *Opts { type OptsFn func(opts *Opts) -func WithMaxCount(maxCount uint) OptsFn { +func WithMaxRetryCount(maxCount uint) OptsFn { return func(opts *Opts) { - opts.maxCount = maxCount + if maxCount == 0 { + maxCount = defaultMaxRetryCount + } + + opts.maxRetryCount = maxCount } } diff --git a/registry/exec/exec_test.go b/registry/exec/exec_test.go new file mode 100644 index 000000000..dc1971182 --- /dev/null +++ b/registry/exec/exec_test.go @@ -0,0 +1,153 @@ +package exec + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestRetryableExecutor_ExecWithFallback(t *testing.T) { + exec := NewRetryableExecutor[int]() + + execFnCallCount := 0 + fallbackFnCallCount := 0 + + execFnRes := 11 + + res, err := exec.ExecWithFallback(func() (int, error) { + execFnCallCount++ + + return execFnRes, nil + }, func() error { + fallbackFnCallCount++ + + return nil + }) + + assert.Nil(t, err) + assert.Equal(t, execFnRes, res) + assert.Equal(t, 1, execFnCallCount) + assert.Equal(t, 0, fallbackFnCallCount) +} + +func TestRetryableExecutor_ExecWithFallback_RetrySuccess(t *testing.T) { + exec := NewRetryableExecutor[int]() + + execFnCallCount := 0 + fallbackFnCallCount := 0 + + execFnRes := 11 + + res, err := exec.ExecWithFallback(func() (int, error) { + execFnCallCount++ + + if execFnCallCount < 2 { + return 0, errors.New("boom") + } + + return execFnRes, nil + }, func() error { + fallbackFnCallCount++ + + return nil + }) + + assert.Nil(t, err) + assert.Equal(t, execFnRes, res) + assert.Equal(t, 2, execFnCallCount) + assert.Equal(t, 1, fallbackFnCallCount) +} + +func TestRetryableExecutor_ExecWithFallback_NilFns(t *testing.T) { + exec := NewRetryableExecutor[int]() + + res, err := exec.ExecWithFallback(nil, nil) + assert.ErrorIs(t, err, ErrMissingExecFn) + assert.Equal(t, 0, res) + + res, err = exec.ExecWithFallback(func() (int, error) { + return 1, nil + }, nil) + assert.ErrorIs(t, err, ErrMissingFallbackFn) + assert.Equal(t, 0, res) + +} + +func TestRetryableExecutor_ExecWithFallback_ExecFnError(t *testing.T) { + retryCount := uint(5) + + exec := NewRetryableExecutor[int]( + WithMaxRetryCount(retryCount), + WithErrTimeout(100*time.Millisecond), + ) + + execFnCallCount := uint(0) + fallbackFnCallCount := uint(0) + + res, err := exec.ExecWithFallback(func() (int, error) { + execFnCallCount++ + + return 0, errors.New("boom") + }, func() error { + fallbackFnCallCount++ + + return nil + }) + assert.NotNil(t, err) + assert.Equal(t, 0, res) + assert.Equal(t, retryCount+1, execFnCallCount) + assert.Equal(t, retryCount, fallbackFnCallCount) + + execErr := err.(*Error) + assert.Len(t, execErr.errs, int(retryCount+1)) +} + +func TestRetryableExecutor_ExecWithFallback_FallBackFnError(t *testing.T) { + exec := NewRetryableExecutor[int]() + + execFnCallCount := 0 + fallbackFnCallCount := 0 + + res, err := exec.ExecWithFallback(func() (int, error) { + execFnCallCount++ + + return 0, errors.New("boom") + }, func() error { + fallbackFnCallCount++ + + return errors.New("boom") + }) + assert.NotNil(t, err) + assert.Equal(t, 0, res) + assert.Equal(t, defaultMaxRetryCount+1, execFnCallCount) + assert.Equal(t, defaultMaxRetryCount, fallbackFnCallCount) + + execErr := err.(*Error) + assert.Len(t, execErr.errs, defaultMaxRetryCount+1) +} + +func TestRetryableExecutor_ExecWithFallback_FallBackFnError_NoRetry(t *testing.T) { + exec := NewRetryableExecutor[int](WithRetryOnFallBackError(false)) + + execFnCallCount := 0 + fallbackFnCallCount := 0 + + res, err := exec.ExecWithFallback(func() (int, error) { + execFnCallCount++ + + return 0, errors.New("boom") + }, func() error { + fallbackFnCallCount++ + + return errors.New("boom") + }) + assert.NotNil(t, err) + assert.Equal(t, 0, res) + assert.Equal(t, 1, execFnCallCount) + assert.Equal(t, 1, fallbackFnCallCount) + + execErr := err.(*Error) + assert.Len(t, execErr.errs, 2) +} diff --git a/registry/factory_mock.go b/registry/factory_mock.go new file mode 100644 index 000000000..2c8b918ba --- /dev/null +++ b/registry/factory_mock.go @@ -0,0 +1,97 @@ +// Code generated by mockery v2.13.0-beta.1. DO NOT EDIT. + +package registry + +import ( + types "github.com/centrifuge/go-substrate-rpc-client/v4/types" + mock "github.com/stretchr/testify/mock" +) + +// FactoryMock is an autogenerated mock type for the Factory type +type FactoryMock struct { + mock.Mock +} + +// CreateCallRegistry provides a mock function with given fields: meta +func (_m *FactoryMock) CreateCallRegistry(meta *types.Metadata) (CallRegistry, error) { + ret := _m.Called(meta) + + var r0 CallRegistry + if rf, ok := ret.Get(0).(func(*types.Metadata) CallRegistry); ok { + r0 = rf(meta) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(CallRegistry) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*types.Metadata) error); ok { + r1 = rf(meta) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CreateErrorRegistry provides a mock function with given fields: meta +func (_m *FactoryMock) CreateErrorRegistry(meta *types.Metadata) (ErrorRegistry, error) { + ret := _m.Called(meta) + + var r0 ErrorRegistry + if rf, ok := ret.Get(0).(func(*types.Metadata) ErrorRegistry); ok { + r0 = rf(meta) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(ErrorRegistry) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*types.Metadata) error); ok { + r1 = rf(meta) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CreateEventRegistry provides a mock function with given fields: meta +func (_m *FactoryMock) CreateEventRegistry(meta *types.Metadata) (EventRegistry, error) { + ret := _m.Called(meta) + + var r0 EventRegistry + if rf, ok := ret.Get(0).(func(*types.Metadata) EventRegistry); ok { + r0 = rf(meta) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(EventRegistry) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*types.Metadata) error); ok { + r1 = rf(meta) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type NewFactoryMockT interface { + mock.TestingT + Cleanup(func()) +} + +// NewFactoryMock creates a new instance of FactoryMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewFactoryMock(t NewFactoryMockT) *FactoryMock { + mock := &FactoryMock{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/registry/parser/parser.go b/registry/parser/parser.go index 02b957de4..74b6e4438 100644 --- a/registry/parser/parser.go +++ b/registry/parser/parser.go @@ -26,7 +26,7 @@ type EventParser interface { } type eventParser struct { - stateProvider state.StateProvider + stateProvider state.Provider registryFactory registry.Factory eventStorageExecutor exec.RetryableExecutor[*types.StorageDataRaw] @@ -37,7 +37,7 @@ type eventParser struct { } func NewParser( - stateProvider state.StateProvider, + stateProvider state.Provider, registryFactory registry.Factory, eventStorageExecutor exec.RetryableExecutor[*types.StorageDataRaw], eventParsingExecutor exec.RetryableExecutor[[]*Event], @@ -56,9 +56,9 @@ func NewParser( return parser, nil } -func NewDefaultParser(stateProvider state.StateProvider, registryFactory registry.Factory) (EventParser, error) { +func NewDefaultParser(stateProvider state.Provider, registryFactory registry.Factory) (EventParser, error) { eventStorageExecutor := exec.NewRetryableExecutor[*types.StorageDataRaw](exec.WithErrTimeout(1 * time.Second)) - eventParsingExecutor := exec.NewRetryableExecutor[[]*Event](exec.WithMaxCount(1)) + eventParsingExecutor := exec.NewRetryableExecutor[[]*Event](exec.WithMaxRetryCount(1)) return NewParser(stateProvider, registryFactory, eventStorageExecutor, eventParsingExecutor) } diff --git a/registry/parser/parser_test.go b/registry/parser/parser_integration_test.go similarity index 78% rename from registry/parser/parser_test.go rename to registry/parser/parser_integration_test.go index ae15ee505..0c6057e25 100644 --- a/registry/parser/parser_test.go +++ b/registry/parser/parser_integration_test.go @@ -1,3 +1,5 @@ +//go:build integration + package parser import ( @@ -39,7 +41,7 @@ func TestParser_GetEvents(t *testing.T) { return } - parser, err := NewDefaultParser(state.NewStateProvider(api.RPC.State), registry.NewFactory()) + parser, err := NewDefaultParser(state.NewProvider(api.RPC.State), registry.NewFactory()) if err != nil { log.Printf("Couldn't create eventParser: %s", err) @@ -55,11 +57,13 @@ func TestParser_GetEvents(t *testing.T) { var previousBlock *types.SignedBlock + processedBlockCount := 0 + for { block, err := api.RPC.Chain.GetBlock(blockHash) if err != nil { - log.Printf("Skipping block for '%s': %s\n", testURL, err) + log.Printf("Skipping block %d for '%s' due to a block retrieval error\n", previousBlock.Block.Header.Number-1, testURL) if blockHash, err = api.RPC.Chain.GetBlockHash(uint64(previousBlock.Block.Header.Number - 2)); err != nil { log.Printf("Couldn't get block hash for block %d: %s", previousBlock.Block.Header.Number-2, err) @@ -80,6 +84,12 @@ func TestParser_GetEvents(t *testing.T) { log.Printf("Couldn't parse events for '%s', block number %d: %s\n", testURL, block.Block.Header.Number, err) return } + + processedBlockCount++ + + if processedBlockCount%500 == 0 { + log.Printf("Parsed events for %d blocks for '%s' so far, last block number %d\n", processedBlockCount, testURL, block.Block.Header.Number) + } } }() } diff --git a/registry/registry.go b/registry/registry.go index aa646d47a..564bd60ba 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -9,6 +9,8 @@ import ( "github.com/centrifuge/go-substrate-rpc-client/v4/types" ) +//go:generate mockery --name Factory --structname FactoryMock --filename factory_mock.go --inpackage + // Factory is the interface responsible for generating the according registries from the metadata. type Factory interface { CreateCallRegistry(meta *types.Metadata) (CallRegistry, error) diff --git a/registry/state/state.go b/registry/state/provider.go similarity index 60% rename from registry/state/state.go rename to registry/state/provider.go index 474e509cd..2df3a871a 100644 --- a/registry/state/state.go +++ b/registry/state/provider.go @@ -5,26 +5,28 @@ import ( "github.com/centrifuge/go-substrate-rpc-client/v4/types" ) -type StateProvider interface { +//go:generate mockery --name Provider --structname ProviderMock --filename provider_mock.go --inpackage + +type Provider interface { GetMetadata(blockHash types.Hash) (*types.Metadata, error) GetLatestMetadata() (*types.Metadata, error) GetStorageEvents(meta *types.Metadata, blockHash types.Hash) (*types.StorageDataRaw, error) } -type stateProvider struct { +type provider struct { stateRPC state.State } -func NewStateProvider(stateRPC state.State) StateProvider { - return &stateProvider{stateRPC: stateRPC} +func NewProvider(stateRPC state.State) Provider { + return &provider{stateRPC: stateRPC} } -func (s *stateProvider) GetLatestMetadata() (*types.Metadata, error) { +func (s *provider) GetLatestMetadata() (*types.Metadata, error) { return s.stateRPC.GetMetadataLatest() } -func (s *stateProvider) GetMetadata(blockHash types.Hash) (*types.Metadata, error) { +func (s *provider) GetMetadata(blockHash types.Hash) (*types.Metadata, error) { return s.stateRPC.GetMetadata(blockHash) } @@ -33,7 +35,7 @@ const ( storageMethod = "Events" ) -func (s *stateProvider) GetStorageEvents(meta *types.Metadata, blockHash types.Hash) (*types.StorageDataRaw, error) { +func (s *provider) GetStorageEvents(meta *types.Metadata, blockHash types.Hash) (*types.StorageDataRaw, error) { key, err := types.CreateStorageKey(meta, storagePrefix, storageMethod, nil) if err != nil { diff --git a/registry/state/provider_mock.go b/registry/state/provider_mock.go new file mode 100644 index 000000000..476096c9c --- /dev/null +++ b/registry/state/provider_mock.go @@ -0,0 +1,97 @@ +// Code generated by mockery v2.13.0-beta.1. DO NOT EDIT. + +package state + +import ( + types "github.com/centrifuge/go-substrate-rpc-client/v4/types" + mock "github.com/stretchr/testify/mock" +) + +// ProviderMock is an autogenerated mock type for the Provider type +type ProviderMock struct { + mock.Mock +} + +// GetLatestMetadata provides a mock function with given fields: +func (_m *ProviderMock) GetLatestMetadata() (*types.Metadata, error) { + ret := _m.Called() + + var r0 *types.Metadata + if rf, ok := ret.Get(0).(func() *types.Metadata); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Metadata) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetMetadata provides a mock function with given fields: blockHash +func (_m *ProviderMock) GetMetadata(blockHash types.Hash) (*types.Metadata, error) { + ret := _m.Called(blockHash) + + var r0 *types.Metadata + if rf, ok := ret.Get(0).(func(types.Hash) *types.Metadata); ok { + r0 = rf(blockHash) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.Metadata) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(types.Hash) error); ok { + r1 = rf(blockHash) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetStorageEvents provides a mock function with given fields: meta, blockHash +func (_m *ProviderMock) GetStorageEvents(meta *types.Metadata, blockHash types.Hash) (*types.StorageDataRaw, error) { + ret := _m.Called(meta, blockHash) + + var r0 *types.StorageDataRaw + if rf, ok := ret.Get(0).(func(*types.Metadata, types.Hash) *types.StorageDataRaw); ok { + r0 = rf(meta, blockHash) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*types.StorageDataRaw) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*types.Metadata, types.Hash) error); ok { + r1 = rf(meta, blockHash) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +type NewProviderMockT interface { + mock.TestingT + Cleanup(func()) +} + +// NewProviderMock creates a new instance of ProviderMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +func NewProviderMock(t NewProviderMockT) *ProviderMock { + mock := &ProviderMock{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/registry/state/provider_test.go b/registry/state/provider_test.go new file mode 100644 index 000000000..8a3f40dc3 --- /dev/null +++ b/registry/state/provider_test.go @@ -0,0 +1,103 @@ +package state + +import ( + "errors" + "testing" + + "github.com/centrifuge/go-substrate-rpc-client/v4/types/codec" + + "github.com/centrifuge/go-substrate-rpc-client/v4/rpc/state/mocks" + "github.com/centrifuge/go-substrate-rpc-client/v4/types" + "github.com/stretchr/testify/assert" +) + +func TestProvider_GetLatestMetadata(t *testing.T) { + stateRPCMock := mocks.NewState(t) + + provider := NewProvider(stateRPCMock) + + testMeta := &types.Metadata{} + + stateRPCMock.On("GetMetadataLatest"). + Return(testMeta, nil). + Once() + + res, err := provider.GetLatestMetadata() + assert.NoError(t, err) + assert.Equal(t, testMeta, res) + + stateRPCError := errors.New("error") + + stateRPCMock.On("GetMetadataLatest"). + Return(nil, stateRPCError). + Once() + + res, err = provider.GetLatestMetadata() + assert.ErrorIs(t, err, stateRPCError) + assert.Nil(t, res) +} + +func TestProvider_GetMetadata(t *testing.T) { + stateRPCMock := mocks.NewState(t) + + provider := NewProvider(stateRPCMock) + + testHash := types.Hash{} + testMeta := &types.Metadata{} + + stateRPCMock.On("GetMetadata", testHash). + Return(testMeta, nil). + Once() + + res, err := provider.GetMetadata(testHash) + assert.NoError(t, err) + assert.Equal(t, testMeta, res) + + stateRPCError := errors.New("error") + + stateRPCMock.On("GetMetadata", testHash). + Return(nil, stateRPCError). + Once() + + res, err = provider.GetMetadata(testHash) + assert.ErrorIs(t, err, stateRPCError) + assert.Nil(t, res) + + types.NewMetadataV14() +} + +func TestProvider_GetStorageEvents(t *testing.T) { + stateRPCMock := mocks.NewState(t) + + provider := NewProvider(stateRPCMock) + + testHash := types.Hash{} + + var testMeta types.Metadata + + err := codec.DecodeFromHex(types.MetadataV14Data, &testMeta) + assert.NoError(t, err) + + storageKey, err := types.CreateStorageKey(&testMeta, storagePrefix, storageMethod, nil) + assert.NoError(t, err) + + storageData := &types.StorageDataRaw{} + + stateRPCMock.On("GetStorageRaw", storageKey, testHash). + Return(storageData, nil). + Once() + + res, err := provider.GetStorageEvents(&testMeta, testHash) + assert.NoError(t, err) + assert.Equal(t, storageData, res) + + stateRPCError := errors.New("error") + + stateRPCMock.On("GetStorageRaw", storageKey, testHash). + Return(nil, stateRPCError). + Once() + + res, err = provider.GetStorageEvents(&testMeta, testHash) + assert.ErrorIs(t, err, stateRPCError) + assert.Nil(t, res) +}