From b43c46f48189926713c58a9760168ef2496c3bff Mon Sep 17 00:00:00 2001 From: Rian Hughes Date: Wed, 16 Oct 2024 15:42:07 +0300 Subject: [PATCH] Support plugins (#2051) Co-authored-by: rian Co-authored-by: LordGhostX --- README.md | 1 + blockchain/blockchain.go | 18 +++++- cmd/juno/juno.go | 4 ++ core/state.go | 64 ++++++++++++++------- docs/docs/plugins.md | 82 ++++++++++++++++++++++++++ docs/sidebars.js | 1 + go.mod | 1 + go.sum | 2 + mocks/mock_plugin.go | 98 ++++++++++++++++++++++++++++++++ node/node.go | 14 +++++ plugin/plugin.go | 73 ++++++++++++++++++++++++ plugin/plugin_test.go | 79 +++++++++++++++++++++++++ sync/sync.go | 65 +++++++++++++++++++-- vm/rust/src/juno_state_reader.rs | 15 +++-- 14 files changed, 487 insertions(+), 30 deletions(-) create mode 100644 docs/docs/plugins.md create mode 100644 mocks/mock_plugin.go create mode 100644 plugin/plugin.go create mode 100644 plugin/plugin_test.go diff --git a/README.md b/README.md index bed33114d8..3eaeab1df9 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,7 @@ After following these steps, Juno should be up and running on your machine, util - Starknet state construction and storage using a path-based Merkle Patricia trie. - Feeder gateway synchronisation of Blocks, Transactions, Receipts, State Updates and Classes. - Block and Transaction hash verification. +- Plugins ## 🛣 Roadmap diff --git a/blockchain/blockchain.go b/blockchain/blockchain.go index ee2411c471..4aa6659b98 100644 --- a/blockchain/blockchain.go +++ b/blockchain/blockchain.go @@ -828,6 +828,23 @@ func (b *Blockchain) RevertHead() error { return b.database.Update(b.revertHead) } +func (b *Blockchain) GetReverseStateDiff() (*core.StateDiff, error) { + var reverseStateDiff *core.StateDiff + return reverseStateDiff, b.database.View(func(txn db.Transaction) error { + blockNumber, err := chainHeight(txn) + if err != nil { + return err + } + stateUpdate, err := stateUpdateByNumber(txn, blockNumber) + if err != nil { + return err + } + state := core.NewState(txn) + reverseStateDiff, err = state.GetReverseStateDiff(blockNumber, stateUpdate.StateDiff) + return err + }) +} + func (b *Blockchain) revertHead(txn db.Transaction) error { blockNumber, err := chainHeight(txn) if err != nil { @@ -874,7 +891,6 @@ func (b *Blockchain) revertHead(txn db.Transaction) error { } // Revert chain height and pending. - if genesisBlock { if err = txn.Delete(db.Pending.Key()); err != nil { return err diff --git a/cmd/juno/juno.go b/cmd/juno/juno.go index c61ee40529..6739a63357 100644 --- a/cmd/juno/juno.go +++ b/cmd/juno/juno.go @@ -82,6 +82,7 @@ const ( callMaxStepsF = "rpc-call-max-steps" corsEnableF = "rpc-cors-enable" versionedConstantsFileF = "versioned-constants-file" + pluginPathF = "plugin-path" defaultConfig = "" defaulHost = "localhost" @@ -119,6 +120,7 @@ const ( defaultGwTimeout = 5 * time.Second defaultCorsEnable = false defaultVersionedConstantsFile = "" + defaultPluginPath = "" configFlagUsage = "The YAML configuration file." logLevelFlagUsage = "Options: trace, debug, info, warn, error." @@ -170,6 +172,7 @@ const ( "The upper limit is 4 million steps, and any higher value will still be capped at 4 million." corsEnableUsage = "Enable CORS on RPC endpoints" versionedConstantsFileUsage = "Use custom versioned constants from provided file" + pluginPathUsage = "Path to the plugin .so file" ) var Version string @@ -355,6 +358,7 @@ func NewCmd(config *node.Config, run func(*cobra.Command, []string) error) *cobr junoCmd.Flags().Bool(corsEnableF, defaultCorsEnable, corsEnableUsage) junoCmd.Flags().String(versionedConstantsFileF, defaultVersionedConstantsFile, versionedConstantsFileUsage) junoCmd.MarkFlagsMutuallyExclusive(p2pFeederNodeF, p2pPeersF) + junoCmd.Flags().String(pluginPathF, defaultPluginPath, pluginPathUsage) junoCmd.AddCommand(GenP2PKeyPair(), DBCmd(defaultDBPath)) diff --git a/core/state.go b/core/state.go index 71198afbe6..effde8b518 100644 --- a/core/state.go +++ b/core/state.go @@ -540,10 +540,14 @@ func (s *State) Revert(blockNumber uint64, update *StateUpdate) error { return fmt.Errorf("remove declared classes: %v", err) } - // update contracts - reversedDiff, err := s.buildReverseDiff(blockNumber, update.StateDiff) + reversedDiff, err := s.GetReverseStateDiff(blockNumber, update.StateDiff) if err != nil { - return fmt.Errorf("build reverse diff: %v", err) + return fmt.Errorf("error getting reverse state diff: %v", err) + } + + err = s.performStateDeletions(blockNumber, update.StateDiff) + if err != nil { + return fmt.Errorf("error performing state deletions: %v", err) } stateTrie, storageCloser, err := s.storage() @@ -566,12 +570,17 @@ func (s *State) Revert(blockNumber uint64, update *StateUpdate) error { } } - // purge noClassContracts - // + if err = s.purgeNoClassContracts(); err != nil { + return err + } + + return s.verifyStateUpdateRoot(update.OldRoot) +} + +func (s *State) purgeNoClassContracts() error { // As noClassContracts are not in StateDiff.DeployedContracts we can only purge them if their storage no longer exists. // Updating contracts with reverse diff will eventually lead to the deletion of noClassContract's storage key from db. Thus, // we can use the lack of key's existence as reason for purging noClassContracts. - for addr := range noClassContracts { noClassC, err := NewContractUpdater(&addr, s.txn) if err != nil { @@ -592,8 +601,7 @@ func (s *State) Revert(blockNumber uint64, update *StateUpdate) error { } } } - - return s.verifyStateUpdateRoot(update.OldRoot) + return nil } func (s *State) removeDeclaredClasses(blockNumber uint64, v0Classes []*felt.Felt, v1Classes map[felt.Felt]*felt.Felt) error { @@ -657,7 +665,7 @@ func (s *State) purgeContract(addr *felt.Felt) error { return storageCloser() } -func (s *State) buildReverseDiff(blockNumber uint64, diff *StateDiff) (*StateDiff, error) { +func (s *State) GetReverseStateDiff(blockNumber uint64, diff *StateDiff) (*StateDiff, error) { reversed := *diff // storage diffs @@ -673,10 +681,6 @@ func (s *State) buildReverseDiff(blockNumber uint64, diff *StateDiff) (*StateDif } value = oldValue } - - if err := s.DeleteContractStorageLog(&addr, &key, blockNumber); err != nil { - return nil, err - } reversedDiffs[key] = value } reversed.StorageDiffs[addr] = reversedDiffs @@ -686,7 +690,6 @@ func (s *State) buildReverseDiff(blockNumber uint64, diff *StateDiff) (*StateDif reversed.Nonces = make(map[felt.Felt]*felt.Felt, len(diff.Nonces)) for addr := range diff.Nonces { oldNonce := &felt.Zero - if blockNumber > 0 { var err error oldNonce, err = s.ContractNonceAt(&addr, blockNumber-1) @@ -694,10 +697,6 @@ func (s *State) buildReverseDiff(blockNumber uint64, diff *StateDiff) (*StateDif return nil, err } } - - if err := s.DeleteContractNonceLog(&addr, blockNumber); err != nil { - return nil, err - } reversed.Nonces[addr] = oldNonce } @@ -712,12 +711,35 @@ func (s *State) buildReverseDiff(blockNumber uint64, diff *StateDiff) (*StateDif return nil, err } } + reversed.ReplacedClasses[addr] = classHash + } + return &reversed, nil +} + +func (s *State) performStateDeletions(blockNumber uint64, diff *StateDiff) error { + // storage diffs + for addr, storageDiffs := range diff.StorageDiffs { + for key := range storageDiffs { + if err := s.DeleteContractStorageLog(&addr, &key, blockNumber); err != nil { + return err + } + } + } + + // nonces + for addr := range diff.Nonces { + if err := s.DeleteContractNonceLog(&addr, blockNumber); err != nil { + return err + } + } + + // replaced classes + for addr := range diff.ReplacedClasses { if err := s.DeleteContractClassHashLog(&addr, blockNumber); err != nil { - return nil, err + return err } - reversed.ReplacedClasses[addr] = classHash } - return &reversed, nil + return nil } diff --git a/docs/docs/plugins.md b/docs/docs/plugins.md new file mode 100644 index 0000000000..816f028906 --- /dev/null +++ b/docs/docs/plugins.md @@ -0,0 +1,82 @@ +--- +title: Juno Plugins +--- + +Juno supports plugins that satisfy the `JunoPlugin` interface, enabling developers to extend and customize Juno's behaviour and functionality by dynamically loading external plugins during runtime. + +The `JunoPlugin` interface provides a structured way for plugins to interact with the blockchain by sending notifications when new blocks are added or reverted. This ensures state consistency, especially during blockchain reorganizations, while abstracting away the complexity of implementing block syncing and revert logic. + +## JunoPlugin Interface + +Your plugin must implement the `JunoPlugin` interface, which includes methods for initializing, shutting down, and handling new and reverted blocks. + +```go +type JunoPlugin interface { + Init() error + Shutdown() error + NewBlock(block *core.Block, stateUpdate *core.StateUpdate, newClasses map[felt.Felt]core.Class) error + RevertBlock(from, to *BlockAndStateUpdate, reverseStateDiff *core.StateDiff) error +} +``` + +**Init**: Called when the plugin is initialized. This can be used to set up database connections or any other necessary resources. + +**Shutdown**: Called when the Juno node is shut down. This can be used to clean up resources like database connections. + +**NewBlock**: Triggered when a new block is synced by the Juno client. Juno will send the block, the corresponding state update, and any new classes. Importantly, Juno waits for the plugin to finish processing this function call before continuing. This ensures that the plugin completes its task before Juno proceeds with the blockchain sync. + +**RevertBlock**: Called during a blockchain reorganization (reorg). Juno will invoke this method for each block that needs to be reverted. Similar to NewBlock, the client will wait for the plugin to finish handling the revert before moving on to the next block. + +## Example plugin + +Here is a basic example of a plugin that satisfies the `JunoPlugin` interface: + +```go +// go:generate go build -buildmode=plugin -o ../../build/plugin.so ./example.go +type examplePlugin string + +// Important: "JunoPluginInstance" needs to be exported for Juno to load the plugin correctly +var JunoPluginInstance examplePlugin + +var _ junoplugin.JunoPlugin = (*examplePlugin)(nil) + +func (p *examplePlugin) Init() error { + fmt.Println("ExamplePlugin initialized") + return nil +} + +func (p *examplePlugin) Shutdown() error { + fmt.Println("ExamplePlugin shutdown") + return nil +} + +func (p *examplePlugin) NewBlock(block *core.Block, stateUpdate *core.StateUpdate, newClasses map[felt.Felt]core.Class) error { + fmt.Println("ExamplePlugin NewBlock called") + return nil +} + +func (p *examplePlugin) RevertBlock(from, to *junoplugin.BlockAndStateUpdate, reverseStateDiff *core.StateDiff) error { + fmt.Println("ExamplePlugin RevertBlock called") + return nil +} +``` + +The `JunoPluginInstance` variable must be exported for Juno to correctly load the plugin: +`var JunoPluginInstance examplePlugin` + +We ensure the plugin implements the `JunoPlugin` interface, with the following line: +`var _ junoplugin.JunoPlugin = (*examplePlugin)(nil)` + +## Building and loading the plugin + +Once you have written your plugin, you can compile it into a shared object file (.so) using the following command: + +```shell +go build -buildmode=plugin -o ./plugin.so /path/to/your/plugin.go +``` + +This command compiles the plugin into a shared object file (`plugin.so`), which can then be loaded by the Juno client using the `--plugin-path` flag. + +## Running Juno with the plugin + +Once your plugin has been compiled into a `.so` file, you can run Juno with your plugin by providing the `--plugin-path` flag. This flag tells Juno where to find and load your plugin at runtime. diff --git a/docs/sidebars.js b/docs/sidebars.js index 10125a3020..b98fd7bacf 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -12,6 +12,7 @@ const sidebars = { "hardware-requirements", "running-juno", "configuring", + "plugins", "running-on-gcp", "updating", ], diff --git a/go.mod b/go.mod index df47107f31..56016a4cbf 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/ethereum/go-ethereum v1.14.11 github.com/fxamacker/cbor/v2 v2.7.0 github.com/go-playground/validator/v10 v10.22.1 + github.com/golang/protobuf v1.5.4 github.com/jinzhu/copier v0.4.0 github.com/libp2p/go-libp2p v0.36.2 github.com/libp2p/go-libp2p-kad-dht v0.27.0 diff --git a/go.sum b/go.sum index 32c31eb96b..0481e512ca 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= diff --git a/mocks/mock_plugin.go b/mocks/mock_plugin.go new file mode 100644 index 0000000000..7c1d4a4391 --- /dev/null +++ b/mocks/mock_plugin.go @@ -0,0 +1,98 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/NethermindEth/juno/plugin (interfaces: JunoPlugin) +// +// Generated by this command: +// +// mockgen -destination=../mocks/mock_plugin.go -package=mocks github.com/NethermindEth/juno/plugin JunoPlugin +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + core "github.com/NethermindEth/juno/core" + felt "github.com/NethermindEth/juno/core/felt" + junoplugin "github.com/NethermindEth/juno/plugin" + gomock "go.uber.org/mock/gomock" +) + +// MockJunoPlugin is a mock of JunoPlugin interface. +type MockJunoPlugin struct { + ctrl *gomock.Controller + recorder *MockJunoPluginMockRecorder +} + +// MockJunoPluginMockRecorder is the mock recorder for MockJunoPlugin. +type MockJunoPluginMockRecorder struct { + mock *MockJunoPlugin +} + +// NewMockJunoPlugin creates a new mock instance. +func NewMockJunoPlugin(ctrl *gomock.Controller) *MockJunoPlugin { + mock := &MockJunoPlugin{ctrl: ctrl} + mock.recorder = &MockJunoPluginMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockJunoPlugin) EXPECT() *MockJunoPluginMockRecorder { + return m.recorder +} + +// Init mocks base method. +func (m *MockJunoPlugin) Init() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Init") + ret0, _ := ret[0].(error) + return ret0 +} + +// Init indicates an expected call of Init. +func (mr *MockJunoPluginMockRecorder) Init() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Init", reflect.TypeOf((*MockJunoPlugin)(nil).Init)) +} + +// NewBlock mocks base method. +func (m *MockJunoPlugin) NewBlock(arg0 *core.Block, arg1 *core.StateUpdate, arg2 map[felt.Felt]core.Class) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewBlock", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// NewBlock indicates an expected call of NewBlock. +func (mr *MockJunoPluginMockRecorder) NewBlock(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewBlock", reflect.TypeOf((*MockJunoPlugin)(nil).NewBlock), arg0, arg1, arg2) +} + +// RevertBlock mocks base method. +func (m *MockJunoPlugin) RevertBlock(arg0, arg1 *junoplugin.BlockAndStateUpdate, arg2 *core.StateDiff) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RevertBlock", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RevertBlock indicates an expected call of RevertBlock. +func (mr *MockJunoPluginMockRecorder) RevertBlock(arg0, arg1, arg2 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RevertBlock", reflect.TypeOf((*MockJunoPlugin)(nil).RevertBlock), arg0, arg1, arg2) +} + +// Shutdown mocks base method. +func (m *MockJunoPlugin) Shutdown() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Shutdown") + ret0, _ := ret[0].(error) + return ret0 +} + +// Shutdown indicates an expected call of Shutdown. +func (mr *MockJunoPluginMockRecorder) Shutdown() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockJunoPlugin)(nil).Shutdown)) +} diff --git a/node/node.go b/node/node.go index 123a5b7085..96d94a182d 100644 --- a/node/node.go +++ b/node/node.go @@ -22,6 +22,7 @@ import ( "github.com/NethermindEth/juno/l1" "github.com/NethermindEth/juno/migration" "github.com/NethermindEth/juno/p2p" + junoplugin "github.com/NethermindEth/juno/plugin" "github.com/NethermindEth/juno/rpc" "github.com/NethermindEth/juno/service" adaptfeeder "github.com/NethermindEth/juno/starknetdata/feeder" @@ -88,6 +89,8 @@ type Config struct { GatewayAPIKey string `mapstructure:"gw-api-key"` GatewayTimeout time.Duration `mapstructure:"gw-timeout"` + + PluginPath string `mapstructure:"plugin-path"` } type Node struct { @@ -156,6 +159,17 @@ func New(cfg *Config, version string) (*Node, error) { //nolint:gocyclo,funlen synchronizer := sync.New(chain, adaptfeeder.New(client), log, cfg.PendingPollInterval, dbIsRemote) gatewayClient := gateway.NewClient(cfg.Network.GatewayURL, log).WithUserAgent(ua).WithAPIKey(cfg.GatewayAPIKey) + pluginService := junoplugin.New(log) + if cfg.PluginPath != "" { + plugin, err := junoplugin.Load(cfg.PluginPath) + if err != nil { + return nil, err + } + synchronizer.WithPlugin(plugin) + pluginService.WithPlugin(plugin) + services = append(services, pluginService) + } + var p2pService *p2p.Service if cfg.P2P { if cfg.Network != utils.Sepolia { diff --git a/plugin/plugin.go b/plugin/plugin.go new file mode 100644 index 0000000000..cb18f70343 --- /dev/null +++ b/plugin/plugin.go @@ -0,0 +1,73 @@ +package junoplugin + +import ( + "context" + "fmt" + "plugin" + "sync" + + "github.com/NethermindEth/juno/core" + "github.com/NethermindEth/juno/core/felt" + "github.com/NethermindEth/juno/utils" +) + +type PluginService struct { + jPlugin JunoPlugin + wg sync.WaitGroup + log utils.SimpleLogger +} + +func New(log utils.SimpleLogger) *PluginService { + return &PluginService{wg: sync.WaitGroup{}, log: log} +} + +func (p *PluginService) WithPlugin(jPlugin JunoPlugin) { + p.jPlugin = jPlugin +} + +func (p *PluginService) Run(ctx context.Context) error { + p.wg.Add(1) + go func() { + defer p.wg.Done() + <-ctx.Done() + if err := p.jPlugin.Shutdown(); err != nil { + p.log.Errorw("Error while calling plugin Shutdown() function", "err", err) + } + }() + p.wg.Wait() + return nil +} + +//go:generate mockgen -destination=../mocks/mock_plugin.go -package=mocks github.com/NethermindEth/juno/plugin JunoPlugin +type JunoPlugin interface { + Init() error + Shutdown() error + NewBlock(block *core.Block, stateUpdate *core.StateUpdate, newClasses map[felt.Felt]core.Class) error + // The state is reverted by applying a write operation with the reverseStateDiff's StorageDiffs, Nonces, and ReplacedClasses, + // and a delete option with its DeclaredV0Classes, DeclaredV1Classes, and ReplacedClasses. + RevertBlock(from, to *BlockAndStateUpdate, reverseStateDiff *core.StateDiff) error +} + +type BlockAndStateUpdate struct { + Block *core.Block + StateUpdate *core.StateUpdate +} + +func Load(pluginPath string) (JunoPlugin, error) { + plug, err := plugin.Open(pluginPath) + if err != nil { + return nil, fmt.Errorf("error loading plugin .so file: %w", err) + } + + symPlugin, err := plug.Lookup("JunoPluginInstance") + if err != nil { + return nil, fmt.Errorf("error looking up PluginInstance: %w", err) + } + + pluginInstance, ok := symPlugin.(JunoPlugin) + if !ok { + return nil, fmt.Errorf("the plugin does not staisfy the required interface") + } + + return pluginInstance, pluginInstance.Init() +} diff --git a/plugin/plugin_test.go b/plugin/plugin_test.go new file mode 100644 index 0000000000..54f9cffe17 --- /dev/null +++ b/plugin/plugin_test.go @@ -0,0 +1,79 @@ +package junoplugin_test + +import ( + "context" + "testing" + "time" + + "github.com/NethermindEth/juno/blockchain" + "github.com/NethermindEth/juno/clients/feeder" + "github.com/NethermindEth/juno/db/pebble" + "github.com/NethermindEth/juno/mocks" + junoplugin "github.com/NethermindEth/juno/plugin" + adaptfeeder "github.com/NethermindEth/juno/starknetdata/feeder" + "github.com/NethermindEth/juno/sync" + "github.com/NethermindEth/juno/utils" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestPlugin(t *testing.T) { + timeout := time.Second + mockCtrl := gomock.NewController(t) + t.Cleanup(mockCtrl.Finish) + + plugin := mocks.NewMockJunoPlugin(mockCtrl) + + mainClient := feeder.NewTestClient(t, &utils.Mainnet) + mainGw := adaptfeeder.New(mainClient) + + integClient := feeder.NewTestClient(t, &utils.Integration) + integGw := adaptfeeder.New(integClient) + + testDB := pebble.NewMemTest(t) + + // sync to integration for 2 blocks + for i := range 2 { + su, block, err := integGw.StateUpdateWithBlock(context.Background(), uint64(i)) + require.NoError(t, err) + plugin.EXPECT().NewBlock(block, su, gomock.Any()) + } + bc := blockchain.New(testDB, &utils.Integration) + synchronizer := sync.New(bc, integGw, utils.NewNopZapLogger(), 0, false).WithPlugin(plugin) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + require.NoError(t, synchronizer.Run(ctx)) + cancel() + + t.Run("resync to mainnet with the same db", func(t *testing.T) { + bc := blockchain.New(testDB, &utils.Mainnet) + + // Ensure current head is Integration head + head, err := bc.HeadsHeader() + require.NoError(t, err) + require.Equal(t, utils.HexToFelt(t, "0x34e815552e42c5eb5233b99de2d3d7fd396e575df2719bf98e7ed2794494f86"), head.Hash) + + // Reorg 2 blocks, then sync 3 blocks + su1, block1, err := integGw.StateUpdateWithBlock(context.Background(), uint64(1)) + require.NoError(t, err) + su0, block0, err := integGw.StateUpdateWithBlock(context.Background(), uint64(0)) + require.NoError(t, err) + plugin.EXPECT().RevertBlock(&junoplugin.BlockAndStateUpdate{block1, su1}, &junoplugin.BlockAndStateUpdate{block0, su0}, gomock.Any()) + plugin.EXPECT().RevertBlock(&junoplugin.BlockAndStateUpdate{block0, su0}, nil, gomock.Any()) + for i := range 3 { + su, block, err := mainGw.StateUpdateWithBlock(context.Background(), uint64(i)) + require.NoError(t, err) + plugin.EXPECT().NewBlock(block, su, gomock.Any()) + } + + synchronizer = sync.New(bc, mainGw, utils.NewNopZapLogger(), 0, false).WithPlugin(plugin) + ctx, cancel = context.WithTimeout(context.Background(), timeout) + require.NoError(t, synchronizer.Run(ctx)) + cancel() + + // After syncing (and reorging) the current head should be at mainnet + head, err = bc.HeadsHeader() + require.NoError(t, err) + require.Equal(t, utils.HexToFelt(t, "0x4e1f77f39545afe866ac151ac908bd1a347a2a8a7d58bef1276db4f06fdf2f6"), head.Hash) + }) +} diff --git a/sync/sync.go b/sync/sync.go index a2e4ac0bf0..1270193d07 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -12,6 +12,7 @@ import ( "github.com/NethermindEth/juno/core/felt" "github.com/NethermindEth/juno/db" "github.com/NethermindEth/juno/feed" + junoplugin "github.com/NethermindEth/juno/plugin" "github.com/NethermindEth/juno/service" "github.com/NethermindEth/juno/starknetdata" "github.com/NethermindEth/juno/utils" @@ -72,6 +73,7 @@ type Synchronizer struct { pendingPollInterval time.Duration catchUpMode bool + plugin junoplugin.JunoPlugin } func New(bc *blockchain.Blockchain, starkNetData starknetdata.StarknetData, @@ -89,6 +91,12 @@ func New(bc *blockchain.Blockchain, starkNetData starknetdata.StarknetData, return s } +// WithPlugin registers an plugin +func (s *Synchronizer) WithPlugin(plugin junoplugin.JunoPlugin) *Synchronizer { + s.plugin = plugin + return s +} + // WithListener registers an EventListener func (s *Synchronizer) WithListener(listener EventListener) *Synchronizer { s.listener = listener @@ -180,6 +188,49 @@ func (s *Synchronizer) fetchUnknownClasses(ctx context.Context, stateUpdate *cor return newClasses, closer() } +func (s *Synchronizer) handlePluginRevertBlock() { + fromBlock, err := s.blockchain.Head() + if err != nil { + s.log.Warnw("Failed to retrieve the reverted blockchain head block for the plugin", "err", err) + return + } + fromSU, err := s.blockchain.StateUpdateByNumber(fromBlock.Number) + if err != nil { + s.log.Warnw("Failed to retrieve the reverted blockchain head state-update for the plugin", "err", err) + return + } + reverseStateDiff, err := s.blockchain.GetReverseStateDiff() + if err != nil { + s.log.Warnw("Failed to retrieve reverse state diff", "head", fromBlock.Number, "hash", fromBlock.Hash.ShortString(), "err", err) + return + } + + var toBlockAndStateUpdate *junoplugin.BlockAndStateUpdate + if fromBlock.Number != 0 { + toBlock, err := s.blockchain.BlockByHash(fromBlock.ParentHash) + if err != nil { + s.log.Warnw("Failed to retrieve the parent block for the plugin", "err", err) + return + } + toSU, err := s.blockchain.StateUpdateByNumber(toBlock.Number) + if err != nil { + s.log.Warnw("Failed to retrieve the parents state-update for the plugin", "err", err) + return + } + toBlockAndStateUpdate = &junoplugin.BlockAndStateUpdate{ + Block: toBlock, + StateUpdate: toSU, + } + } + err = s.plugin.RevertBlock( + &junoplugin.BlockAndStateUpdate{Block: fromBlock, StateUpdate: fromSU}, + toBlockAndStateUpdate, + reverseStateDiff) + if err != nil { + s.log.Errorw("Plugin RevertBlock failure:", "err", err) + } +} + func (s *Synchronizer) verifierTask(ctx context.Context, block *core.Block, stateUpdate *core.StateUpdate, newClasses map[felt.Felt]core.Class, resetStreams context.CancelFunc, ) stream.Callback { @@ -205,6 +256,9 @@ func (s *Synchronizer) verifierTask(ctx context.Context, block *core.Block, stat // revert the head and restart the sync process, hoping that the reorg is not deep // if the reorg is deeper, we will end up here again and again until we fully revert reorged // blocks + if s.plugin != nil { + s.handlePluginRevertBlock() + } s.revertHead(block) } else { s.log.Warnw("Failed storing Block", "number", block.Number, @@ -231,6 +285,12 @@ func (s *Synchronizer) verifierTask(ctx context.Context, block *core.Block, stat s.newHeads.Send(block.Header) s.log.Infow("Stored Block", "number", block.Number, "hash", block.Hash.ShortString(), "root", block.GlobalStateRoot.ShortString()) + if s.plugin != nil { + err := s.plugin.NewBlock(block, stateUpdate, newClasses) + if err != nil { + s.log.Errorw("Plugin NewBlock failure:", err) + } + } } } } @@ -315,11 +375,8 @@ func (s *Synchronizer) revertHead(forkBlock *core.Block) { if err == nil { localHead = head.Hash } - s.log.Infow("Reorg detected", "localHead", localHead, "forkHead", forkBlock.Hash) - - err = s.blockchain.RevertHead() - if err != nil { + if err := s.blockchain.RevertHead(); err != nil { s.log.Warnw("Failed reverting HEAD", "reverted", localHead, "err", err) } else { s.log.Infow("Reverted HEAD", "reverted", localHead) diff --git a/vm/rust/src/juno_state_reader.rs b/vm/rust/src/juno_state_reader.rs index df540005d7..ee4ad9734d 100644 --- a/vm/rust/src/juno_state_reader.rs +++ b/vm/rust/src/juno_state_reader.rs @@ -1,5 +1,5 @@ use std::{ - ffi::{c_char, c_uchar, c_void, c_int, CStr}, + ffi::{c_char, c_int, c_uchar, c_void, CStr}, slice, sync::Mutex, }; @@ -75,8 +75,14 @@ impl StateReader for JunoStateReader { let addr = felt_to_byte_array(contract_address.0.key()); let storage_key = felt_to_byte_array(key.0.key()); let mut buffer: [u8; 32] = [0; 32]; - let wrote = - unsafe { JunoStateGetStorageAt(self.handle, addr.as_ptr(), storage_key.as_ptr(), buffer.as_mut_ptr()) }; + let wrote = unsafe { + JunoStateGetStorageAt( + self.handle, + addr.as_ptr(), + storage_key.as_ptr(), + buffer.as_mut_ptr(), + ) + }; if wrote == 0 { Err(StateError::StateReadError(format!( "failed to read location {} at address {}", @@ -111,7 +117,8 @@ impl StateReader for JunoStateReader { fn get_class_hash_at(&self, contract_address: ContractAddress) -> StateResult { let addr = felt_to_byte_array(contract_address.0.key()); let mut buffer: [u8; 32] = [0; 32]; - let wrote = unsafe { JunoStateGetClassHashAt(self.handle, addr.as_ptr(), buffer.as_mut_ptr()) }; + let wrote = + unsafe { JunoStateGetClassHashAt(self.handle, addr.as_ptr(), buffer.as_mut_ptr()) }; if wrote == 0 { Err(StateError::StateReadError(format!( "failed to read class hash of address {}",