From 3ce6ff878b60e6c119925cd78ddb62dd49b09080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Toni=20Ram=C3=ADrez?= <58293609+ToniRamirezM@users.noreply.github.com> Date: Tue, 30 Apr 2024 17:22:13 +0200 Subject: [PATCH] Update DS Format (#3608) * protobuf datastream --- Makefile | 1 + .../src/proto/datastream/v1/datastream.proto | 64 ++ sequencer/batch.go | 6 + sequencer/datastreamer.go | 27 +- sequencer/finalizer_test.go | 1 + sequencer/forcedbatch.go | 2 +- sequencer/l2block.go | 5 +- sequencer/sequencer.go | 134 ++- state/datastream.go | 482 +++++------ state/datastream/datastream.pb.go | 773 ++++++++++++++++++ state/pgstatestorage/datastream.go | 13 +- state/test/datastream_test.go | 82 -- test/docker-compose.yml | 6 + tools/datastreamer/Makefile | 22 +- tools/datastreamer/config/tool.config.toml | 10 +- tools/datastreamer/main.go | 733 +++++++---------- 16 files changed, 1541 insertions(+), 820 deletions(-) create mode 100644 proto/src/proto/datastream/v1/datastream.proto create mode 100644 state/datastream/datastream.pb.go delete mode 100644 state/test/datastream_test.go diff --git a/Makefile b/Makefile index 83d4fed5fe..ba55fb3429 100644 --- a/Makefile +++ b/Makefile @@ -164,6 +164,7 @@ generate-code-from-proto: ## Generates code from proto files cd proto/src/proto/hashdb/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../merkletree/hashdb --go-grpc_out=../../../../../merkletree/hashdb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative hashdb.proto cd proto/src/proto/executor/v1 && protoc --proto_path=. --go_out=../../../../../state/runtime/executor --go-grpc_out=../../../../../state/runtime/executor --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative executor.proto cd proto/src/proto/aggregator/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../aggregator/prover --go-grpc_out=../../../../../aggregator/prover --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative aggregator.proto + cd proto/src/proto/datastream/v1 && protoc --proto_path=. --proto_path=../../../../include --go_out=../../../../../state/datastream --go-grpc_out=../../../../../state/datastream --go-grpc_opt=paths=source_relative --go_opt=paths=source_relative datastream.proto ## Help display. ## Pulls comments from beside commands and prints a nicely formatted diff --git a/proto/src/proto/datastream/v1/datastream.proto b/proto/src/proto/datastream/v1/datastream.proto new file mode 100644 index 0000000000..6b8a89adc6 --- /dev/null +++ b/proto/src/proto/datastream/v1/datastream.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +package datastream.v1; + +option go_package = "github.com/0xPolygonHermez/zkevm-node/state/datastream"; + +message Batch { + uint64 number = 1; + bytes local_exit_root = 2; + bytes state_root = 3; + uint64 fork_id = 4; + uint64 chain_id = 5; +} + +message L2Block { + uint64 number = 1; + uint64 batch_number = 2; + uint64 timestamp = 3; + uint32 delta_timestamp = 4; + uint64 min_timestamp = 5; + bytes l1_blockhash = 6; + uint32 l1_infotree_index = 7; + bytes hash = 8; + bytes state_root = 9; + bytes global_exit_root = 10; + bytes coinbase = 11; +} + +message Transaction { + uint64 l2block_number = 1; + bool is_valid = 2; + bytes encoded = 3; + uint32 effective_gas_price_percentage = 4; + bytes im_state_root = 5; +} + +message UpdateGER { + uint64 batch_number = 1; + uint64 timestamp = 2; + bytes global_exit_root = 3; + bytes coinbase = 4; + uint64 fork_id = 5; + uint64 chain_id = 6; + bytes state_root = 7; +} + +message BookMark { + BookmarkType type = 1; + uint64 value = 2; +} + +enum BookmarkType { + BOOKMARK_TYPE_UNSPECIFIED = 0; + BOOKMARK_TYPE_BATCH = 1; + BOOKMARK_TYPE_L2_BLOCK = 2; +} + +enum EntryType { + ENTRY_TYPE_UNSPECIFIED = 0; + ENTRY_TYPE_BATCH = 1; + ENTRY_TYPE_L2_BLOCK = 2; + ENTRY_TYPE_TRANSACTION = 3; + ENTRY_TYPE_UPDATE_GER = 4; +} diff --git a/sequencer/batch.go b/sequencer/batch.go index ecaa93aca7..8db3320a4d 100644 --- a/sequencer/batch.go +++ b/sequencer/batch.go @@ -29,6 +29,7 @@ type Batch struct { finalRemainingResources state.BatchResources // remaining batch resources when a L2 block is processed finalHighReservedZKCounters state.ZKCounters closingReason state.ClosingReason + finalLocalExitRoot common.Hash } func (b *Batch) isEmpty() bool { @@ -99,6 +100,7 @@ func (f *finalizer) setWIPBatch(ctx context.Context, wipStateBatch *state.Batch) finalRemainingResources: remainingResources, imHighReservedZKCounters: wipStateBatch.HighReservedZKCounters, finalHighReservedZKCounters: wipStateBatch.HighReservedZKCounters, + finalLocalExitRoot: wipStateBatch.LocalExitRoot, } return wipBatch, nil @@ -312,6 +314,7 @@ func (f *finalizer) openNewWIPBatch(batchNumber uint64, stateRoot common.Hash) * imRemainingResources: maxRemainingResources, finalRemainingResources: maxRemainingResources, closingReason: state.EmptyClosingReason, + finalLocalExitRoot: state.ZeroHash, } } @@ -400,6 +403,9 @@ func (f *finalizer) closeSIPBatch(ctx context.Context, dbTx pgx.Tx) error { }() } + // Sent batch to DS + f.DSSendBatch(f.wipBatch.batchNumber, f.wipBatch.finalStateRoot, f.wipBatch.finalLocalExitRoot) + log.Infof("sip batch %d closed in statedb, closing reason: %s", f.sipBatch.batchNumber, f.sipBatch.closingReason) f.sipBatch = nil diff --git a/sequencer/datastreamer.go b/sequencer/datastreamer.go index 7f5e7e763a..81ed1083b0 100644 --- a/sequencer/datastreamer.go +++ b/sequencer/datastreamer.go @@ -3,9 +3,11 @@ package sequencer import ( "github.com/0xPolygonHermez/zkevm-node/log" "github.com/0xPolygonHermez/zkevm-node/state" + "github.com/0xPolygonHermez/zkevm-node/state/datastream" + "github.com/ethereum/go-ethereum/common" ) -func (f *finalizer) DSSendL2Block(batchNumber uint64, blockResponse *state.ProcessBlockResponse, l1InfoTreeIndex uint32) error { +func (f *finalizer) DSSendL2Block(batchNumber uint64, blockResponse *state.ProcessBlockResponse, l1InfoTreeIndex uint32, minTimestamp uint64) error { forkID := f.stateIntf.GetForkIDByBatchNumber(batchNumber) // Send data to streamer @@ -13,12 +15,13 @@ func (f *finalizer) DSSendL2Block(batchNumber uint64, blockResponse *state.Proce l2Block := state.DSL2Block{ BatchNumber: batchNumber, L2BlockNumber: blockResponse.BlockNumber, - Timestamp: int64(blockResponse.Timestamp), + Timestamp: blockResponse.Timestamp, + Min_timestamp: minTimestamp, L1InfoTreeIndex: l1InfoTreeIndex, L1BlockHash: blockResponse.BlockHashL1, GlobalExitRoot: blockResponse.GlobalExitRoot, Coinbase: f.sequencerAddress, - ForkID: uint16(forkID), + ForkID: forkID, BlockHash: blockResponse.BlockHash, StateRoot: blockResponse.BlockHash, //From etrog, the blockhash is the block root } @@ -57,9 +60,23 @@ func (f *finalizer) DSSendBatchBookmark(batchNumber uint64) { // Check if stream server enabled if f.streamServer != nil { // Send batch bookmark to the streamer - f.dataToStream <- state.DSBookMark{ - Type: state.BookMarkTypeBatch, + f.dataToStream <- datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_BATCH, Value: batchNumber, } } } + +func (f *finalizer) DSSendBatch(batchNumber uint64, stateRoot common.Hash, localExitRoot common.Hash) { + forkID := f.stateIntf.GetForkIDByBatchNumber(batchNumber) + + if f.streamServer != nil { + // Send batch to the streamer + f.dataToStream <- datastream.Batch{ + Number: batchNumber, + ForkId: forkID, + StateRoot: stateRoot.Bytes(), + LocalExitRoot: localExitRoot.Bytes(), + } + } +} diff --git a/sequencer/finalizer_test.go b/sequencer/finalizer_test.go index 6e54c342f8..84a2d6e8cf 100644 --- a/sequencer/finalizer_test.go +++ b/sequencer/finalizer_test.go @@ -980,6 +980,7 @@ func TestFinalizer_finalizeSIPBatch(t *testing.T) { // arrange stateMock.On("BeginStateTransaction", ctx).Return(dbTxMock, nilErr).Once() + stateMock.On("GetForkIDByBatchNumber", mock.Anything).Return(uint64(state.FORKID_BLUEBERRY)) stateMock.On("CloseWIPBatch", ctx, receipt, mock.Anything).Return(tc.managerErr).Once() if tc.managerErr == nil { diff --git a/sequencer/forcedbatch.go b/sequencer/forcedbatch.go index 85f74abee1..21a0c92504 100644 --- a/sequencer/forcedbatch.go +++ b/sequencer/forcedbatch.go @@ -198,7 +198,7 @@ func (f *finalizer) handleProcessForcedBatchResponse(ctx context.Context, newBat } // Send L2 block to data streamer - err = f.DSSendL2Block(newBatchNumber, forcedL2BlockResponse, 0) + err = f.DSSendL2Block(newBatchNumber, forcedL2BlockResponse, 0, forcedL2BlockResponse.Timestamp) if err != nil { //TODO: we need to halt/rollback the L2 block if we had an error sending to the data streamer? log.Errorf("error sending L2 block %d to data streamer, error: %v", forcedL2BlockResponse.BlockNumber, err) diff --git a/sequencer/l2block.go b/sequencer/l2block.go index b64a6db902..f045ef2764 100644 --- a/sequencer/l2block.go +++ b/sequencer/l2block.go @@ -259,8 +259,9 @@ func (f *finalizer) processL2Block(ctx context.Context, l2Block *L2Block) error return fmt.Errorf(overflowLog) } - // Update finalStateRoot of the batch to the newStateRoot for the L2 block + // Update finalStateRoot/finalLocalExitRoot of the batch to the newStateRoot/newLocalExitRoot for the L2 block l2Block.batch.finalStateRoot = l2Block.batchResponse.NewStateRoot + l2Block.batch.finalLocalExitRoot = l2Block.batchResponse.NewLocalExitRoot f.updateFlushIDs(batchResponse.FlushID, batchResponse.StoredFlushID) @@ -491,7 +492,7 @@ func (f *finalizer) storeL2Block(ctx context.Context, l2Block *L2Block) error { log.Infof("[ds-debug] l2 block %d [%d] transactions updated as selected in the pooldb", blockResponse.BlockNumber, l2Block.trackingNum) // Send L2 block to data streamer - err = f.DSSendL2Block(f.wipBatch.batchNumber, blockResponse, l2Block.getL1InfoTreeIndex()) + err = f.DSSendL2Block(f.wipBatch.batchNumber, blockResponse, l2Block.getL1InfoTreeIndex(), l2Block.timestamp) if err != nil { //TODO: we need to halt/rollback the L2 block if we had an error sending to the data streamer? log.Errorf("error sending L2 block %d [%d] to data streamer, error: %v", blockResponse.BlockNumber, l2Block.trackingNum, err) diff --git a/sequencer/sequencer.go b/sequencer/sequencer.go index b79ad26c17..d1945234d5 100644 --- a/sequencer/sequencer.go +++ b/sequencer/sequencer.go @@ -11,7 +11,9 @@ import ( "github.com/0xPolygonHermez/zkevm-node/log" "github.com/0xPolygonHermez/zkevm-node/pool" "github.com/0xPolygonHermez/zkevm-node/state" + "github.com/0xPolygonHermez/zkevm-node/state/datastream" "github.com/ethereum/go-ethereum/common" + "google.golang.org/protobuf/proto" ) const ( @@ -137,7 +139,7 @@ func (s *Sequencer) checkStateInconsistency(ctx context.Context) { } func (s *Sequencer) updateDataStreamerFile(ctx context.Context, chainID uint64) { - err := state.GenerateDataStreamerFile(ctx, s.streamServer, s.stateIntf, true, nil, chainID, s.cfg.StreamServer.UpgradeEtrogBatchNumber) + err := state.GenerateDataStreamFile(ctx, s.streamServer, s.stateIntf, true, nil, chainID, s.cfg.StreamServer.UpgradeEtrogBatchNumber) if err != nil { log.Fatalf("failed to generate data streamer file, error: %v", err) } @@ -278,54 +280,77 @@ func (s *Sequencer) sendDataToStreamer(chainID uint64) { continue } - bookMark := state.DSBookMark{ - Type: state.BookMarkTypeL2Block, + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK, Value: l2Block.L2BlockNumber, } //TODO: remove this log log.Infof("[ds-debug] add stream bookmark for l2block %d", l2Block.L2BlockNumber) - _, err = s.streamServer.AddStreamBookmark(bookMark.Encode()) + marshalledBookMark, err := proto.Marshal(bookMark) + if err != nil { + log.Errorf("failed to marshal bookmark for l2block %d, error: %v", l2Block.L2BlockNumber, err) + continue + } + + _, err = s.streamServer.AddStreamBookmark(marshalledBookMark) if err != nil { log.Errorf("failed to add stream bookmark for l2block %d, error: %v", l2Block.L2BlockNumber, err) continue } // Get previous block timestamp to calculate delta timestamp - previousL2Block := state.DSL2BlockStart{} + previousL2Block := datastream.L2Block{} if l2Block.L2BlockNumber > 0 { - bookMark = state.DSBookMark{ - Type: state.BookMarkTypeL2Block, + bookMark = &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK, Value: l2Block.L2BlockNumber - 1, } //TODO: remove this log log.Infof("[ds-debug] get previous l2block %d", l2Block.L2BlockNumber-1) - previousL2BlockEntry, err := s.streamServer.GetFirstEventAfterBookmark(bookMark.Encode()) + marshalledBookMark, err := proto.Marshal(bookMark) + if err != nil { + log.Errorf("failed to marshal bookmark for l2block %d, error: %v", l2Block.L2BlockNumber, err) + continue + } + + previousL2BlockEntry, err := s.streamServer.GetFirstEventAfterBookmark(marshalledBookMark) if err != nil { log.Errorf("failed to get previous l2block %d, error: %v", l2Block.L2BlockNumber-1, err) continue } - previousL2Block = state.DSL2BlockStart{}.Decode(previousL2BlockEntry.Data) + err = proto.Unmarshal(previousL2BlockEntry.Data, &previousL2Block) + if err != nil { + log.Errorf("failed to unmarshal previous l2block %d, error: %v", l2Block.L2BlockNumber-1, err) + continue + } } - blockStart := state.DSL2BlockStart{ + streamL2Block := &datastream.L2Block{ + Number: l2Block.L2BlockNumber, BatchNumber: l2Block.BatchNumber, - L2BlockNumber: l2Block.L2BlockNumber, Timestamp: l2Block.Timestamp, DeltaTimestamp: uint32(l2Block.Timestamp - previousL2Block.Timestamp), - L1InfoTreeIndex: l2Block.L1InfoTreeIndex, - L1BlockHash: l2Block.L1BlockHash, - GlobalExitRoot: l2Block.GlobalExitRoot, - Coinbase: l2Block.Coinbase, - ForkID: l2Block.ForkID, - ChainID: uint32(chainID), + MinTimestamp: l2Block.Min_timestamp, + L1Blockhash: l2Block.L1BlockHash.Bytes(), + L1InfotreeIndex: l2Block.L1InfoTreeIndex, + Hash: l2Block.BlockHash.Bytes(), + StateRoot: l2Block.StateRoot.Bytes(), + GlobalExitRoot: l2Block.GlobalExitRoot.Bytes(), + Coinbase: l2Block.Coinbase.Bytes(), + } + + marshalledL2Block, err := proto.Marshal(streamL2Block) + if err != nil { + log.Errorf("failed to marshal l2block %d, error: %v", l2Block.L2BlockNumber, err) + continue } //TODO: remove this log log.Infof("[ds-debug] add l2blockStart stream entry for l2block %d", l2Block.L2BlockNumber) - _, err = s.streamServer.AddStreamEntry(state.EntryTypeL2BlockStart, blockStart.Encode()) + _, err = s.streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_L2_BLOCK), marshalledL2Block) if err != nil { log.Errorf("failed to add stream entry for l2block %d, error: %v", l2Block.L2BlockNumber, err) continue @@ -334,25 +359,25 @@ func (s *Sequencer) sendDataToStreamer(chainID uint64) { //TODO: remove this log log.Infof("[ds-debug] adding l2tx stream entries for l2block %d", l2Block.L2BlockNumber) for _, l2Transaction := range l2Block.Txs { - _, err = s.streamServer.AddStreamEntry(state.EntryTypeL2Tx, l2Transaction.Encode()) + streamL2Transaction := &datastream.Transaction{ + L2BlockNumber: l2Transaction.L2BlockNumber, + IsValid: l2Transaction.IsValid != 0, + Encoded: l2Transaction.Encoded, + EffectiveGasPricePercentage: uint32(l2Transaction.EffectiveGasPricePercentage), + ImStateRoot: l2Transaction.ImStateRoot.Bytes(), + } + + marshalledL2Transaction, err := proto.Marshal(streamL2Transaction) if err != nil { - log.Errorf("failed to add l2tx stream entry for l2block %d, error: %v", l2Block.L2BlockNumber, err) + log.Errorf("failed to marshal l2tx for l2block %d, error: %v", l2Block.L2BlockNumber, err) continue } - } - blockEnd := state.DSL2BlockEnd{ - L2BlockNumber: l2Block.L2BlockNumber, - BlockHash: l2Block.BlockHash, - StateRoot: l2Block.StateRoot, - } - - //TODO: remove this log - log.Infof("[ds-debug] add l2blockEnd stream entry for l2block %d", l2Block.L2BlockNumber) - _, err = s.streamServer.AddStreamEntry(state.EntryTypeL2BlockEnd, blockEnd.Encode()) - if err != nil { - log.Errorf("failed to add stream entry for l2block %d, error: %v", l2Block.L2BlockNumber, err) - continue + _, err = s.streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_TRANSACTION), marshalledL2Transaction) + if err != nil { + log.Errorf("failed to add l2tx stream entry for l2block %d, error: %v", l2Block.L2BlockNumber, err) + continue + } } //TODO: remove this log @@ -367,24 +392,55 @@ func (s *Sequencer) sendDataToStreamer(chainID uint64) { log.Infof("[ds-debug] l2block %d sent to datastream", l2Block.L2BlockNumber) // Stream a bookmark - case state.DSBookMark: - bookmark := data + case datastream.BookMark: + err = s.streamServer.StartAtomicOp() + if err != nil { + log.Errorf("failed to start atomic op for bookmark type %d, value %d, error: %v", data.Type, data.Value, err) + continue + } + marshalledBookMark, err := proto.Marshal(&data) + if err != nil { + log.Errorf("failed to marshal bookmark type %d, value %d, error: %v", data.Type, data.Value, err) + continue + } + + _, err = s.streamServer.AddStreamBookmark(marshalledBookMark) + if err != nil { + log.Errorf("failed to add stream bookmark for bookmark type %d, value %d, error: %v", data.Type, data.Value, err) + continue + } + + err = s.streamServer.CommitAtomicOp() + if err != nil { + log.Errorf("failed to commit atomic op for bookmark type %d, value %d, error: %v", data.Type, data.Value, err) + continue + } + case datastream.Batch: err = s.streamServer.StartAtomicOp() if err != nil { - log.Errorf("failed to start atomic op for bookmark type %d, value %d, error: %v", bookmark.Type, bookmark.Value, err) + log.Errorf("failed to start atomic op for batch, error: %v", err) continue } - _, err = s.streamServer.AddStreamBookmark(bookmark.Encode()) + data.ChainId = chainID + + marshalledBatch, err := proto.Marshal(&data) + if err != nil { + log.Errorf("failed to marshal batch, error: %v", err) + continue + } + + _, err = s.streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_BATCH), marshalledBatch) if err != nil { - log.Errorf("failed to add stream bookmark type %d, value %d, error: %v", bookmark.Type, bookmark.Value, err) + log.Errorf("failed to add stream entry for batch, error: %v", err) continue } err = s.streamServer.CommitAtomicOp() if err != nil { - log.Errorf("failed to commit atomic op for bookmark type %d, value %d, error: %v", bookmark.Type, bookmark.Value, err) + log.Errorf("failed to commit atomic op for batch, error: %v", err) + continue } // Invalid stream message type diff --git a/state/datastream.go b/state/datastream.go index d50c7adecf..eb3670e6f9 100644 --- a/state/datastream.go +++ b/state/datastream.go @@ -2,14 +2,16 @@ package state import ( "context" - "encoding/binary" "math/big" + "time" "github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer" "github.com/0xPolygonHermez/zkevm-node/log" + "github.com/0xPolygonHermez/zkevm-node/state/datastream" "github.com/ethereum/go-ethereum/common" "github.com/iden3/go-iden3-crypto/keccak256" "github.com/jackc/pgx/v4" + "google.golang.org/protobuf/proto" ) const ( @@ -17,18 +19,6 @@ const ( StreamTypeSequencer datastreamer.StreamType = 1 // EntryTypeBookMark represents a bookmark entry EntryTypeBookMark datastreamer.EntryType = datastreamer.EtBookmark - // EntryTypeL2BlockStart represents a L2 block start - EntryTypeL2BlockStart datastreamer.EntryType = 1 - // EntryTypeL2Tx represents a L2 transaction - EntryTypeL2Tx datastreamer.EntryType = 2 - // EntryTypeL2BlockEnd represents a L2 block end - EntryTypeL2BlockEnd datastreamer.EntryType = 3 - // EntryTypeUpdateGER represents a GER update - EntryTypeUpdateGER datastreamer.EntryType = 4 - // BookMarkTypeL2Block represents a L2 block bookmark - BookMarkTypeL2Block byte = 0 - // BookMarkTypeBatch represents a batch - BookMarkTypeBatch byte = 1 // SystemSC is the system smart contract address SystemSC = "0x000000000000000000000000000000005ca1ab1e" // posConstant is the constant used to compute the position of the intermediate state root @@ -38,7 +28,8 @@ const ( // DSBatch represents a data stream batch type DSBatch struct { Batch - ForkID uint16 + ForkID uint64 + EtrogTimestamp *time.Time } // DSFullBatch represents a data stream batch ant its L2 blocks @@ -55,64 +46,18 @@ type DSL2FullBlock struct { // DSL2Block is a full l2 block type DSL2Block struct { - BatchNumber uint64 // 8 bytes - L2BlockNumber uint64 // 8 bytes - Timestamp int64 // 8 bytes - L1InfoTreeIndex uint32 // 4 bytes - L1BlockHash common.Hash // 32 bytes - GlobalExitRoot common.Hash // 32 bytes - Coinbase common.Address // 20 bytes - ForkID uint16 // 2 bytes - ChainID uint32 // 4 bytes - BlockHash common.Hash // 32 bytes - StateRoot common.Hash // 32 bytes -} - -// DSL2BlockStart represents a data stream L2 block start -type DSL2BlockStart struct { - BatchNumber uint64 // 8 bytes - L2BlockNumber uint64 // 8 bytes - Timestamp int64 // 8 bytes - DeltaTimestamp uint32 // 4 bytes - L1InfoTreeIndex uint32 // 4 bytes - L1BlockHash common.Hash // 32 bytes - GlobalExitRoot common.Hash // 32 bytes - Coinbase common.Address // 20 bytes - ForkID uint16 // 2 bytes - ChainID uint32 // 4 bytes - -} - -// Encode returns the encoded DSL2BlockStart as a byte slice -func (b DSL2BlockStart) Encode() []byte { - bytes := make([]byte, 0) - bytes = binary.BigEndian.AppendUint64(bytes, b.BatchNumber) - bytes = binary.BigEndian.AppendUint64(bytes, b.L2BlockNumber) - bytes = binary.BigEndian.AppendUint64(bytes, uint64(b.Timestamp)) - bytes = binary.BigEndian.AppendUint32(bytes, b.DeltaTimestamp) - bytes = binary.BigEndian.AppendUint32(bytes, b.L1InfoTreeIndex) - bytes = append(bytes, b.L1BlockHash.Bytes()...) - bytes = append(bytes, b.GlobalExitRoot.Bytes()...) - bytes = append(bytes, b.Coinbase.Bytes()...) - bytes = binary.BigEndian.AppendUint16(bytes, b.ForkID) - bytes = binary.BigEndian.AppendUint32(bytes, b.ChainID) - return bytes -} - -// Decode decodes the DSL2BlockStart from a byte slice -func (b DSL2BlockStart) Decode(data []byte) DSL2BlockStart { - b.BatchNumber = binary.BigEndian.Uint64(data[0:8]) - b.L2BlockNumber = binary.BigEndian.Uint64(data[8:16]) - b.Timestamp = int64(binary.BigEndian.Uint64(data[16:24])) - b.DeltaTimestamp = binary.BigEndian.Uint32(data[24:28]) - b.L1InfoTreeIndex = binary.BigEndian.Uint32(data[28:32]) - b.L1BlockHash = common.BytesToHash(data[32:64]) - b.GlobalExitRoot = common.BytesToHash(data[64:96]) - b.Coinbase = common.BytesToAddress(data[96:116]) - b.ForkID = binary.BigEndian.Uint16(data[116:118]) - b.ChainID = binary.BigEndian.Uint32(data[118:122]) - - return b + BatchNumber uint64 + L2BlockNumber uint64 + Timestamp uint64 + Min_timestamp uint64 + L1InfoTreeIndex uint32 + L1BlockHash common.Hash + GlobalExitRoot common.Hash + Coinbase common.Address + ForkID uint64 + ChainID uint64 + BlockHash common.Hash + StateRoot common.Hash } // DSL2Transaction represents a data stream L2 transaction @@ -126,108 +71,6 @@ type DSL2Transaction struct { Encoded []byte } -// Encode returns the encoded DSL2Transaction as a byte slice -func (l DSL2Transaction) Encode() []byte { - bytes := make([]byte, 0) - bytes = append(bytes, l.EffectiveGasPricePercentage) - bytes = append(bytes, l.IsValid) - bytes = append(bytes, l.StateRoot[:]...) - bytes = binary.BigEndian.AppendUint32(bytes, l.EncodedLength) - bytes = append(bytes, l.Encoded...) - return bytes -} - -// Decode decodes the DSL2Transaction from a byte slice -func (l DSL2Transaction) Decode(data []byte) DSL2Transaction { - l.EffectiveGasPricePercentage = data[0] - l.IsValid = data[1] - l.StateRoot = common.BytesToHash(data[2:34]) - l.EncodedLength = binary.BigEndian.Uint32(data[34:38]) - l.Encoded = data[38:] - return l -} - -// DSL2BlockEnd represents a L2 block end -type DSL2BlockEnd struct { - L2BlockNumber uint64 // 8 bytes - BlockHash common.Hash // 32 bytes - StateRoot common.Hash // 32 bytes -} - -// Encode returns the encoded DSL2BlockEnd as a byte slice -func (b DSL2BlockEnd) Encode() []byte { - bytes := make([]byte, 0) - bytes = binary.BigEndian.AppendUint64(bytes, b.L2BlockNumber) - bytes = append(bytes, b.BlockHash[:]...) - bytes = append(bytes, b.StateRoot[:]...) - return bytes -} - -// Decode decodes the DSL2BlockEnd from a byte slice -func (b DSL2BlockEnd) Decode(data []byte) DSL2BlockEnd { - b.L2BlockNumber = binary.BigEndian.Uint64(data[0:8]) - b.BlockHash = common.BytesToHash(data[8:40]) - b.StateRoot = common.BytesToHash(data[40:72]) - return b -} - -// DSBookMark represents a data stream bookmark -type DSBookMark struct { - Type byte // 1 byte - Value uint64 // 8 bytes -} - -// Encode returns the encoded DSBookMark as a byte slice -func (b DSBookMark) Encode() []byte { - bytes := make([]byte, 0) - bytes = append(bytes, b.Type) - bytes = binary.BigEndian.AppendUint64(bytes, b.Value) - return bytes -} - -// Decode decodes the DSBookMark from a byte slice -func (b DSBookMark) Decode(data []byte) DSBookMark { - b.Type = data[0] - b.Value = binary.BigEndian.Uint64(data[1:9]) - return b -} - -// DSUpdateGER represents a data stream GER update -type DSUpdateGER struct { - BatchNumber uint64 // 8 bytes - Timestamp int64 // 8 bytes - GlobalExitRoot common.Hash // 32 bytes - Coinbase common.Address // 20 bytes - ForkID uint16 // 2 bytes - ChainID uint32 // 4 bytes - StateRoot common.Hash // 32 bytes -} - -// Encode returns the encoded DSUpdateGER as a byte slice -func (g DSUpdateGER) Encode() []byte { - bytes := make([]byte, 0) - bytes = binary.BigEndian.AppendUint64(bytes, g.BatchNumber) - bytes = binary.BigEndian.AppendUint64(bytes, uint64(g.Timestamp)) - bytes = append(bytes, g.GlobalExitRoot[:]...) - bytes = append(bytes, g.Coinbase[:]...) - bytes = binary.BigEndian.AppendUint16(bytes, g.ForkID) - bytes = binary.BigEndian.AppendUint32(bytes, g.ChainID) - bytes = append(bytes, g.StateRoot[:]...) - return bytes -} - -// Decode decodes the DSUpdateGER from a byte slice -func (g DSUpdateGER) Decode(data []byte) DSUpdateGER { - g.BatchNumber = binary.BigEndian.Uint64(data[0:8]) - g.Timestamp = int64(binary.BigEndian.Uint64(data[8:16])) - g.GlobalExitRoot = common.BytesToHash(data[16:48]) - g.Coinbase = common.BytesToAddress(data[48:68]) - g.ForkID = binary.BigEndian.Uint16(data[68:70]) - g.ChainID = binary.BigEndian.Uint32(data[70:74]) - g.StateRoot = common.BytesToHash(data[74:106]) - return g -} - // DSState gathers the methods required to interact with the data stream state. type DSState interface { GetDSGenesisBlock(ctx context.Context, dbTx pgx.Tx) (*DSL2Block, error) @@ -240,14 +83,14 @@ type DSState interface { GetL1InfoRootLeafByIndex(ctx context.Context, l1InfoTreeIndex uint32, dbTx pgx.Tx) (L1InfoTreeExitRootStorageEntry, error) } -// GenerateDataStreamerFile generates or resumes a data stream file -func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.StreamServer, stateDB DSState, readWIPBatch bool, imStateRoots *map[uint64][]byte, chainID uint64, upgradeEtrogBatchNumber uint64) error { +// GenerateDataStreamFile generates or resumes a data stream file +func GenerateDataStreamFile(ctx context.Context, streamServer *datastreamer.StreamServer, stateDB DSState, readWIPBatch bool, imStateRoots *map[uint64][]byte, chainID uint64, upgradeEtrogBatchNumber uint64) error { header := streamServer.GetHeader() var currentBatchNumber uint64 = 0 var lastAddedL2BlockNumber uint64 = 0 var lastAddedBatchNumber uint64 = 0 - var previousTimestamp int64 = 0 + var previousTimestamp uint64 = 0 if header.TotalEntries == 0 { // Get Genesis block @@ -261,52 +104,73 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St return err } - bookMark := DSBookMark{ - Type: BookMarkTypeBatch, + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_BATCH, Value: genesisL2Block.BatchNumber, } - _, err = streamServer.AddStreamBookmark(bookMark.Encode()) + marshalledBookMark, err := proto.Marshal(bookMark) if err != nil { return err } - bookMark = DSBookMark{ - Type: BookMarkTypeL2Block, + _, err = streamServer.AddStreamBookmark(marshalledBookMark) + if err != nil { + return err + } + + bookMark = &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK, Value: genesisL2Block.L2BlockNumber, } - _, err = streamServer.AddStreamBookmark(bookMark.Encode()) + marshalledBookMark, err = proto.Marshal(bookMark) + if err != nil { + return err + } + + _, err = streamServer.AddStreamBookmark(marshalledBookMark) if err != nil { return err } - genesisBlock := DSL2BlockStart{ - BatchNumber: genesisL2Block.BatchNumber, - L2BlockNumber: genesisL2Block.L2BlockNumber, - Timestamp: genesisL2Block.Timestamp, + genesisBlock := &datastream.L2Block{ + Number: genesisL2Block.L2BlockNumber, DeltaTimestamp: 0, - L1InfoTreeIndex: 0, - GlobalExitRoot: genesisL2Block.GlobalExitRoot, - Coinbase: genesisL2Block.Coinbase, - ForkID: genesisL2Block.ForkID, - ChainID: uint32(chainID), + MinTimestamp: 0, + L1InfotreeIndex: 0, + Hash: genesisL2Block.BlockHash.Bytes(), + StateRoot: genesisL2Block.StateRoot.Bytes(), + GlobalExitRoot: genesisL2Block.GlobalExitRoot.Bytes(), + Coinbase: genesisL2Block.Coinbase.Bytes(), } - log.Infof("Genesis block: %+v", genesisBlock) + log.Debugf("Genesis block: %+v", genesisBlock) + + marshalledGenesisBlock, err := proto.Marshal(genesisBlock) + if err != nil { + return err + } - _, err = streamServer.AddStreamEntry(1, genesisBlock.Encode()) + _, err = streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_L2_BLOCK), marshalledGenesisBlock) if err != nil { return err } - genesisBlockEnd := DSL2BlockEnd{ - L2BlockNumber: genesisL2Block.L2BlockNumber, - BlockHash: genesisL2Block.BlockHash, - StateRoot: genesisL2Block.StateRoot, + genesisBatch := &datastream.Batch{ + Number: genesisL2Block.BatchNumber, + LocalExitRoot: common.Hash{}.Bytes(), + StateRoot: genesisL2Block.StateRoot.Bytes(), + ForkId: genesisL2Block.ForkID, + ChainId: chainID, + } + + marshalledGenesisBatch, err := proto.Marshal(genesisBatch) + if err != nil { + return err } - _, err = streamServer.AddStreamEntry(EntryTypeL2BlockEnd, genesisBlockEnd.Encode()) + _, err = streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_BATCH), marshalledGenesisBatch) if err != nil { return err } @@ -325,35 +189,88 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St log.Infof("Latest entry: %+v", latestEntry) switch latestEntry.Type { - case EntryTypeUpdateGER: + case datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_BATCH): + log.Info("Latest entry type is Batch") + + batch := &datastream.Batch{} + if err := proto.Unmarshal(latestEntry.Data, batch); err != nil { + return err + } + + currentBatchNumber = batch.Number + currentBatchNumber++ + case datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_UPDATE_GER): log.Info("Latest entry type is UpdateGER") - currentBatchNumber = binary.BigEndian.Uint64(latestEntry.Data[0:8]) + + updateGer := &datastream.UpdateGER{} + if err := proto.Unmarshal(latestEntry.Data, updateGer); err != nil { + return err + } + + currentBatchNumber = updateGer.BatchNumber currentBatchNumber++ - case EntryTypeL2BlockEnd: - log.Info("Latest entry type is L2BlockEnd") - blockEnd := DSL2BlockEnd{}.Decode(latestEntry.Data) - currentL2BlockNumber := blockEnd.L2BlockNumber - - bookMark := DSBookMark{ - Type: BookMarkTypeL2Block, - Value: currentL2BlockNumber, + case datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_L2_BLOCK): + log.Info("Latest entry type is L2Block") + + l2Block := &datastream.L2Block{} + + if err := proto.Unmarshal(latestEntry.Data, l2Block); err != nil { + return err + } + + currentL2BlockNumber := l2Block.Number + currentBatchNumber = l2Block.BatchNumber + previousTimestamp = l2Block.Timestamp + lastAddedL2BlockNumber = currentL2BlockNumber + case datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_TRANSACTION): + log.Info("Latest entry type is Transaction") + + transaction := &datastream.Transaction{} + if err := proto.Unmarshal(latestEntry.Data, transaction); err != nil { + return err + } + + currentL2BlockNumber := transaction.L2BlockNumber + currentBatchNumber = transaction.L2BlockNumber + lastAddedL2BlockNumber = currentL2BlockNumber + + // Get Previous l2block timestamp + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK, + Value: currentL2BlockNumber - 1, } - firstEntry, err := streamServer.GetFirstEventAfterBookmark(bookMark.Encode()) + marshalledBookMark, err := proto.Marshal(bookMark) if err != nil { return err } - blockStart := DSL2BlockStart{}.Decode(firstEntry.Data) + prevL2BlockEntryNumber, err := streamServer.GetBookmark(marshalledBookMark) + if err != nil { + return err + } + + prevL2BlockEntry, err := streamServer.GetEntry(prevL2BlockEntryNumber) + if err != nil { + return err + } + + prevL2Block := &datastream.L2Block{} + if err := proto.Unmarshal(prevL2BlockEntry.Data, prevL2Block); err != nil { + return err + } + + previousTimestamp = prevL2Block.Timestamp - currentBatchNumber = blockStart.BatchNumber - previousTimestamp = blockStart.Timestamp - lastAddedL2BlockNumber = currentL2BlockNumber case EntryTypeBookMark: log.Info("Latest entry type is BookMark") - bookMark := DSBookMark{} - bookMark = bookMark.Decode(latestEntry.Data) - if bookMark.Type == BookMarkTypeBatch { + + bookMark := &datastream.BookMark{} + if err := proto.Unmarshal(latestEntry.Data, bookMark); err != nil { + return err + } + + if bookMark.Type == datastream.BookmarkType_BOOKMARK_TYPE_BATCH { currentBatchNumber = bookMark.Value } else { log.Fatalf("Latest entry type is an unexpected bookmark type: %v", bookMark.Type) @@ -424,21 +341,26 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St return err } - bookMark := DSBookMark{ - Type: BookMarkTypeBatch, + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_BATCH, Value: batch.BatchNumber, } + marshalledBookMark, err := proto.Marshal(bookMark) + if err != nil { + return err + } + missingBatchBookMark := true if b == 0 { - _, err = streamServer.GetBookmark(bookMark.Encode()) + _, err = streamServer.GetBookmark(marshalledBookMark) if err == nil { missingBatchBookMark = false } } if missingBatchBookMark { - _, err = streamServer.AddStreamBookmark(bookMark.Encode()) + _, err = streamServer.AddStreamBookmark(marshalledBookMark) if err != nil { return err } @@ -448,17 +370,22 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St // Empty batch // Check if there is a GER update if batch.GlobalExitRoot != currentGER && batch.GlobalExitRoot != (common.Hash{}) { - updateGer := DSUpdateGER{ + updateGER := &datastream.UpdateGER{ BatchNumber: batch.BatchNumber, - Timestamp: batch.Timestamp.Unix(), - GlobalExitRoot: batch.GlobalExitRoot, - Coinbase: batch.Coinbase, - ForkID: batch.ForkID, - ChainID: uint32(chainID), - StateRoot: batch.StateRoot, + Timestamp: uint64(batch.Timestamp.Unix()), + GlobalExitRoot: batch.GlobalExitRoot.Bytes(), + Coinbase: batch.Coinbase.Bytes(), + ForkId: batch.ForkID, + ChainId: chainID, + StateRoot: batch.StateRoot.Bytes(), + } + + marshalledUpdateGER, err := proto.Marshal(updateGER) + if err != nil { + return err } - _, err = streamServer.AddStreamEntry(EntryTypeUpdateGER, updateGer.Encode()) + _, err = streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_UPDATE_GER), marshalledUpdateGER) if err != nil { return err } @@ -517,38 +444,57 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St } } - blockStart := DSL2BlockStart{ + streamL2Block := &datastream.L2Block{ + Number: l2Block.L2BlockNumber, BatchNumber: l2Block.BatchNumber, - L2BlockNumber: l2Block.L2BlockNumber, Timestamp: l2Block.Timestamp, DeltaTimestamp: uint32(l2Block.Timestamp - previousTimestamp), - L1InfoTreeIndex: l1InfoTreeIndex, - L1BlockHash: l1BlockHash, - GlobalExitRoot: l2Block.GlobalExitRoot, - Coinbase: l2Block.Coinbase, - ForkID: l2Block.ForkID, - ChainID: uint32(chainID), + MinTimestamp: uint64(batch.Timestamp.Unix()), + L1Blockhash: l1BlockHash.Bytes(), + L1InfotreeIndex: l1InfoTreeIndex, + Hash: l2Block.BlockHash.Bytes(), + StateRoot: l2Block.StateRoot.Bytes(), + GlobalExitRoot: l2Block.GlobalExitRoot.Bytes(), + Coinbase: l2Block.Coinbase.Bytes(), + } + + if l2Block.ForkID >= FORKID_ETROG { + streamL2Block.Hash = l2Block.StateRoot.Bytes() + } + + if l2Block.ForkID == FORKID_ETROG && batch.EtrogTimestamp != nil { + streamL2Block.MinTimestamp = uint64(batch.EtrogTimestamp.Unix()) } previousTimestamp = l2Block.Timestamp - bookMark := DSBookMark{ - Type: BookMarkTypeL2Block, - Value: blockStart.L2BlockNumber, + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK, + Value: streamL2Block.Number, + } + + marshalledBookMark, err := proto.Marshal(bookMark) + if err != nil { + return err } // Check if l2 block was already added - _, err = streamServer.GetBookmark(bookMark.Encode()) + _, err = streamServer.GetBookmark(marshalledBookMark) if err == nil { continue } - _, err = streamServer.AddStreamBookmark(bookMark.Encode()) + _, err = streamServer.AddStreamBookmark(marshalledBookMark) + if err != nil { + return err + } + + marshalledL2Block, err := proto.Marshal(streamL2Block) if err != nil { return err } - _, err = streamServer.AddStreamEntry(EntryTypeL2BlockStart, blockStart.Encode()) + _, err = streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_L2_BLOCK), marshalledL2Block) if err != nil { return err } @@ -559,7 +505,7 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St // > ETROG => IM State root is retrieved from the receipt.im_state_root if l2Block.ForkID < FORKID_ETROG { // Populate intermediate state root with information from the system SC (or cache if available) - if imStateRoots == nil || (*imStateRoots)[blockStart.L2BlockNumber] == nil { + if imStateRoots == nil || (*imStateRoots)[streamL2Block.Number] == nil { position := GetSystemSCPosition(l2Block.L2BlockNumber) imStateRoot, err := stateDB.GetStorageAt(ctx, common.HexToAddress(SystemSC), big.NewInt(0).SetBytes(position), l2Block.StateRoot) if err != nil { @@ -567,35 +513,53 @@ func GenerateDataStreamerFile(ctx context.Context, streamServer *datastreamer.St } tx.StateRoot = common.BigToHash(imStateRoot) } else { - tx.StateRoot = common.BytesToHash((*imStateRoots)[blockStart.L2BlockNumber]) + tx.StateRoot = common.BytesToHash((*imStateRoots)[streamL2Block.Number]) } } else if l2Block.ForkID > FORKID_ETROG { tx.StateRoot = tx.ImStateRoot } - _, err = streamServer.AddStreamEntry(EntryTypeL2Tx, tx.Encode()) + transaction := &datastream.Transaction{ + L2BlockNumber: tx.L2BlockNumber, + IsValid: tx.IsValid != 0, + Encoded: tx.Encoded, + EffectiveGasPricePercentage: uint32(tx.EffectiveGasPricePercentage), + ImStateRoot: tx.StateRoot.Bytes(), + } + + marshalledTransaction, err := proto.Marshal(transaction) if err != nil { return err } - } - - blockEnd := DSL2BlockEnd{ - L2BlockNumber: l2Block.L2BlockNumber, - BlockHash: l2Block.BlockHash, - StateRoot: l2Block.StateRoot, - } - if l2Block.ForkID >= FORKID_ETROG { - blockEnd.BlockHash = l2Block.StateRoot + _, err = streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_TRANSACTION), marshalledTransaction) + if err != nil { + return err + } } - _, err = streamServer.AddStreamEntry(EntryTypeL2BlockEnd, blockEnd.Encode()) - if err != nil { - return err - } currentGER = l2Block.GlobalExitRoot } } + + batch := &datastream.Batch{ + Number: batch.BatchNumber, + LocalExitRoot: batch.LocalExitRoot.Bytes(), + StateRoot: batch.StateRoot.Bytes(), + ForkId: batch.ForkID, + ChainId: chainID, + } + + marshalledBatch, err := proto.Marshal(batch) + if err != nil { + return err + } + + _, err = streamServer.AddStreamEntry(datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_BATCH), marshalledBatch) + if err != nil { + return err + } + // Commit at the end of each batch group err = streamServer.CommitAtomicOp() if err != nil { diff --git a/state/datastream/datastream.pb.go b/state/datastream/datastream.pb.go new file mode 100644 index 0000000000..1f0ede74fc --- /dev/null +++ b/state/datastream/datastream.pb.go @@ -0,0 +1,773 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.32.0 +// protoc v4.25.3 +// source: datastream.proto + +package datastream + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type BookmarkType int32 + +const ( + BookmarkType_BOOKMARK_TYPE_UNSPECIFIED BookmarkType = 0 + BookmarkType_BOOKMARK_TYPE_BATCH BookmarkType = 1 + BookmarkType_BOOKMARK_TYPE_L2_BLOCK BookmarkType = 2 +) + +// Enum value maps for BookmarkType. +var ( + BookmarkType_name = map[int32]string{ + 0: "BOOKMARK_TYPE_UNSPECIFIED", + 1: "BOOKMARK_TYPE_BATCH", + 2: "BOOKMARK_TYPE_L2_BLOCK", + } + BookmarkType_value = map[string]int32{ + "BOOKMARK_TYPE_UNSPECIFIED": 0, + "BOOKMARK_TYPE_BATCH": 1, + "BOOKMARK_TYPE_L2_BLOCK": 2, + } +) + +func (x BookmarkType) Enum() *BookmarkType { + p := new(BookmarkType) + *p = x + return p +} + +func (x BookmarkType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (BookmarkType) Descriptor() protoreflect.EnumDescriptor { + return file_datastream_proto_enumTypes[0].Descriptor() +} + +func (BookmarkType) Type() protoreflect.EnumType { + return &file_datastream_proto_enumTypes[0] +} + +func (x BookmarkType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use BookmarkType.Descriptor instead. +func (BookmarkType) EnumDescriptor() ([]byte, []int) { + return file_datastream_proto_rawDescGZIP(), []int{0} +} + +type EntryType int32 + +const ( + EntryType_ENTRY_TYPE_UNSPECIFIED EntryType = 0 + EntryType_ENTRY_TYPE_BATCH EntryType = 1 + EntryType_ENTRY_TYPE_L2_BLOCK EntryType = 2 + EntryType_ENTRY_TYPE_TRANSACTION EntryType = 3 + EntryType_ENTRY_TYPE_UPDATE_GER EntryType = 4 +) + +// Enum value maps for EntryType. +var ( + EntryType_name = map[int32]string{ + 0: "ENTRY_TYPE_UNSPECIFIED", + 1: "ENTRY_TYPE_BATCH", + 2: "ENTRY_TYPE_L2_BLOCK", + 3: "ENTRY_TYPE_TRANSACTION", + 4: "ENTRY_TYPE_UPDATE_GER", + } + EntryType_value = map[string]int32{ + "ENTRY_TYPE_UNSPECIFIED": 0, + "ENTRY_TYPE_BATCH": 1, + "ENTRY_TYPE_L2_BLOCK": 2, + "ENTRY_TYPE_TRANSACTION": 3, + "ENTRY_TYPE_UPDATE_GER": 4, + } +) + +func (x EntryType) Enum() *EntryType { + p := new(EntryType) + *p = x + return p +} + +func (x EntryType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EntryType) Descriptor() protoreflect.EnumDescriptor { + return file_datastream_proto_enumTypes[1].Descriptor() +} + +func (EntryType) Type() protoreflect.EnumType { + return &file_datastream_proto_enumTypes[1] +} + +func (x EntryType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use EntryType.Descriptor instead. +func (EntryType) EnumDescriptor() ([]byte, []int) { + return file_datastream_proto_rawDescGZIP(), []int{1} +} + +type Batch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Number uint64 `protobuf:"varint,1,opt,name=number,proto3" json:"number,omitempty"` + LocalExitRoot []byte `protobuf:"bytes,2,opt,name=local_exit_root,json=localExitRoot,proto3" json:"local_exit_root,omitempty"` + StateRoot []byte `protobuf:"bytes,3,opt,name=state_root,json=stateRoot,proto3" json:"state_root,omitempty"` + ForkId uint64 `protobuf:"varint,4,opt,name=fork_id,json=forkId,proto3" json:"fork_id,omitempty"` + ChainId uint64 `protobuf:"varint,5,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` +} + +func (x *Batch) Reset() { + *x = Batch{} + if protoimpl.UnsafeEnabled { + mi := &file_datastream_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Batch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Batch) ProtoMessage() {} + +func (x *Batch) ProtoReflect() protoreflect.Message { + mi := &file_datastream_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Batch.ProtoReflect.Descriptor instead. +func (*Batch) Descriptor() ([]byte, []int) { + return file_datastream_proto_rawDescGZIP(), []int{0} +} + +func (x *Batch) GetNumber() uint64 { + if x != nil { + return x.Number + } + return 0 +} + +func (x *Batch) GetLocalExitRoot() []byte { + if x != nil { + return x.LocalExitRoot + } + return nil +} + +func (x *Batch) GetStateRoot() []byte { + if x != nil { + return x.StateRoot + } + return nil +} + +func (x *Batch) GetForkId() uint64 { + if x != nil { + return x.ForkId + } + return 0 +} + +func (x *Batch) GetChainId() uint64 { + if x != nil { + return x.ChainId + } + return 0 +} + +type L2Block struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Number uint64 `protobuf:"varint,1,opt,name=number,proto3" json:"number,omitempty"` + BatchNumber uint64 `protobuf:"varint,2,opt,name=batch_number,json=batchNumber,proto3" json:"batch_number,omitempty"` + Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + DeltaTimestamp uint32 `protobuf:"varint,4,opt,name=delta_timestamp,json=deltaTimestamp,proto3" json:"delta_timestamp,omitempty"` + MinTimestamp uint64 `protobuf:"varint,5,opt,name=min_timestamp,json=minTimestamp,proto3" json:"min_timestamp,omitempty"` + L1Blockhash []byte `protobuf:"bytes,6,opt,name=l1_blockhash,json=l1Blockhash,proto3" json:"l1_blockhash,omitempty"` + L1InfotreeIndex uint32 `protobuf:"varint,7,opt,name=l1_infotree_index,json=l1InfotreeIndex,proto3" json:"l1_infotree_index,omitempty"` + Hash []byte `protobuf:"bytes,8,opt,name=hash,proto3" json:"hash,omitempty"` + StateRoot []byte `protobuf:"bytes,9,opt,name=state_root,json=stateRoot,proto3" json:"state_root,omitempty"` + GlobalExitRoot []byte `protobuf:"bytes,10,opt,name=global_exit_root,json=globalExitRoot,proto3" json:"global_exit_root,omitempty"` + Coinbase []byte `protobuf:"bytes,11,opt,name=coinbase,proto3" json:"coinbase,omitempty"` +} + +func (x *L2Block) Reset() { + *x = L2Block{} + if protoimpl.UnsafeEnabled { + mi := &file_datastream_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *L2Block) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*L2Block) ProtoMessage() {} + +func (x *L2Block) ProtoReflect() protoreflect.Message { + mi := &file_datastream_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use L2Block.ProtoReflect.Descriptor instead. +func (*L2Block) Descriptor() ([]byte, []int) { + return file_datastream_proto_rawDescGZIP(), []int{1} +} + +func (x *L2Block) GetNumber() uint64 { + if x != nil { + return x.Number + } + return 0 +} + +func (x *L2Block) GetBatchNumber() uint64 { + if x != nil { + return x.BatchNumber + } + return 0 +} + +func (x *L2Block) GetTimestamp() uint64 { + if x != nil { + return x.Timestamp + } + return 0 +} + +func (x *L2Block) GetDeltaTimestamp() uint32 { + if x != nil { + return x.DeltaTimestamp + } + return 0 +} + +func (x *L2Block) GetMinTimestamp() uint64 { + if x != nil { + return x.MinTimestamp + } + return 0 +} + +func (x *L2Block) GetL1Blockhash() []byte { + if x != nil { + return x.L1Blockhash + } + return nil +} + +func (x *L2Block) GetL1InfotreeIndex() uint32 { + if x != nil { + return x.L1InfotreeIndex + } + return 0 +} + +func (x *L2Block) GetHash() []byte { + if x != nil { + return x.Hash + } + return nil +} + +func (x *L2Block) GetStateRoot() []byte { + if x != nil { + return x.StateRoot + } + return nil +} + +func (x *L2Block) GetGlobalExitRoot() []byte { + if x != nil { + return x.GlobalExitRoot + } + return nil +} + +func (x *L2Block) GetCoinbase() []byte { + if x != nil { + return x.Coinbase + } + return nil +} + +type Transaction struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + L2BlockNumber uint64 `protobuf:"varint,1,opt,name=l2block_number,json=l2blockNumber,proto3" json:"l2block_number,omitempty"` + IsValid bool `protobuf:"varint,2,opt,name=is_valid,json=isValid,proto3" json:"is_valid,omitempty"` + Encoded []byte `protobuf:"bytes,3,opt,name=encoded,proto3" json:"encoded,omitempty"` + EffectiveGasPricePercentage uint32 `protobuf:"varint,4,opt,name=effective_gas_price_percentage,json=effectiveGasPricePercentage,proto3" json:"effective_gas_price_percentage,omitempty"` + ImStateRoot []byte `protobuf:"bytes,5,opt,name=im_state_root,json=imStateRoot,proto3" json:"im_state_root,omitempty"` +} + +func (x *Transaction) Reset() { + *x = Transaction{} + if protoimpl.UnsafeEnabled { + mi := &file_datastream_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Transaction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Transaction) ProtoMessage() {} + +func (x *Transaction) ProtoReflect() protoreflect.Message { + mi := &file_datastream_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Transaction.ProtoReflect.Descriptor instead. +func (*Transaction) Descriptor() ([]byte, []int) { + return file_datastream_proto_rawDescGZIP(), []int{2} +} + +func (x *Transaction) GetL2BlockNumber() uint64 { + if x != nil { + return x.L2BlockNumber + } + return 0 +} + +func (x *Transaction) GetIsValid() bool { + if x != nil { + return x.IsValid + } + return false +} + +func (x *Transaction) GetEncoded() []byte { + if x != nil { + return x.Encoded + } + return nil +} + +func (x *Transaction) GetEffectiveGasPricePercentage() uint32 { + if x != nil { + return x.EffectiveGasPricePercentage + } + return 0 +} + +func (x *Transaction) GetImStateRoot() []byte { + if x != nil { + return x.ImStateRoot + } + return nil +} + +type UpdateGER struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + BatchNumber uint64 `protobuf:"varint,1,opt,name=batch_number,json=batchNumber,proto3" json:"batch_number,omitempty"` + Timestamp uint64 `protobuf:"varint,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + GlobalExitRoot []byte `protobuf:"bytes,3,opt,name=global_exit_root,json=globalExitRoot,proto3" json:"global_exit_root,omitempty"` + Coinbase []byte `protobuf:"bytes,4,opt,name=coinbase,proto3" json:"coinbase,omitempty"` + ForkId uint64 `protobuf:"varint,5,opt,name=fork_id,json=forkId,proto3" json:"fork_id,omitempty"` + ChainId uint64 `protobuf:"varint,6,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` + StateRoot []byte `protobuf:"bytes,7,opt,name=state_root,json=stateRoot,proto3" json:"state_root,omitempty"` +} + +func (x *UpdateGER) Reset() { + *x = UpdateGER{} + if protoimpl.UnsafeEnabled { + mi := &file_datastream_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *UpdateGER) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UpdateGER) ProtoMessage() {} + +func (x *UpdateGER) ProtoReflect() protoreflect.Message { + mi := &file_datastream_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UpdateGER.ProtoReflect.Descriptor instead. +func (*UpdateGER) Descriptor() ([]byte, []int) { + return file_datastream_proto_rawDescGZIP(), []int{3} +} + +func (x *UpdateGER) GetBatchNumber() uint64 { + if x != nil { + return x.BatchNumber + } + return 0 +} + +func (x *UpdateGER) GetTimestamp() uint64 { + if x != nil { + return x.Timestamp + } + return 0 +} + +func (x *UpdateGER) GetGlobalExitRoot() []byte { + if x != nil { + return x.GlobalExitRoot + } + return nil +} + +func (x *UpdateGER) GetCoinbase() []byte { + if x != nil { + return x.Coinbase + } + return nil +} + +func (x *UpdateGER) GetForkId() uint64 { + if x != nil { + return x.ForkId + } + return 0 +} + +func (x *UpdateGER) GetChainId() uint64 { + if x != nil { + return x.ChainId + } + return 0 +} + +func (x *UpdateGER) GetStateRoot() []byte { + if x != nil { + return x.StateRoot + } + return nil +} + +type BookMark struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type BookmarkType `protobuf:"varint,1,opt,name=type,proto3,enum=datastream.v1.BookmarkType" json:"type,omitempty"` + Value uint64 `protobuf:"varint,2,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *BookMark) Reset() { + *x = BookMark{} + if protoimpl.UnsafeEnabled { + mi := &file_datastream_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BookMark) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BookMark) ProtoMessage() {} + +func (x *BookMark) ProtoReflect() protoreflect.Message { + mi := &file_datastream_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BookMark.ProtoReflect.Descriptor instead. +func (*BookMark) Descriptor() ([]byte, []int) { + return file_datastream_proto_rawDescGZIP(), []int{4} +} + +func (x *BookMark) GetType() BookmarkType { + if x != nil { + return x.Type + } + return BookmarkType_BOOKMARK_TYPE_UNSPECIFIED +} + +func (x *BookMark) GetValue() uint64 { + if x != nil { + return x.Value + } + return 0 +} + +var File_datastream_proto protoreflect.FileDescriptor + +var file_datastream_proto_rawDesc = []byte{ + 0x0a, 0x10, 0x64, 0x61, 0x74, 0x61, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, + 0x31, 0x22, 0x9a, 0x01, 0x0a, 0x05, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6e, + 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x75, 0x6d, + 0x62, 0x65, 0x72, 0x12, 0x26, 0x0a, 0x0f, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x65, 0x78, 0x69, + 0x74, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x45, 0x78, 0x69, 0x74, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x6f, + 0x72, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x66, 0x6f, 0x72, + 0x6b, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x22, 0xf8, + 0x02, 0x0a, 0x07, 0x4c, 0x32, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x6e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x62, 0x61, 0x74, 0x63, 0x68, 0x4e, + 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x12, 0x27, 0x0a, 0x0f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0e, 0x64, 0x65, + 0x6c, 0x74, 0x61, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x23, 0x0a, 0x0d, + 0x6d, 0x69, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0c, 0x6d, 0x69, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x12, 0x21, 0x0a, 0x0c, 0x6c, 0x31, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x68, 0x61, 0x73, + 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x6c, 0x31, 0x42, 0x6c, 0x6f, 0x63, 0x6b, + 0x68, 0x61, 0x73, 0x68, 0x12, 0x2a, 0x0a, 0x11, 0x6c, 0x31, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x74, + 0x72, 0x65, 0x65, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, + 0x0f, 0x6c, 0x31, 0x49, 0x6e, 0x66, 0x6f, 0x74, 0x72, 0x65, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, + 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, + 0x68, 0x61, 0x73, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x72, 0x6f, + 0x6f, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x52, + 0x6f, 0x6f, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x5f, 0x65, 0x78, + 0x69, 0x74, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x67, + 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x45, 0x78, 0x69, 0x74, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x1a, 0x0a, + 0x08, 0x63, 0x6f, 0x69, 0x6e, 0x62, 0x61, 0x73, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x08, 0x63, 0x6f, 0x69, 0x6e, 0x62, 0x61, 0x73, 0x65, 0x22, 0xd2, 0x01, 0x0a, 0x0b, 0x54, 0x72, + 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x32, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x0d, 0x6c, 0x32, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, + 0x12, 0x19, 0x0a, 0x08, 0x69, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x07, 0x69, 0x73, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x65, + 0x6e, 0x63, 0x6f, 0x64, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x65, 0x6e, + 0x63, 0x6f, 0x64, 0x65, 0x64, 0x12, 0x43, 0x0a, 0x1e, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x5f, 0x67, 0x61, 0x73, 0x5f, 0x70, 0x72, 0x69, 0x63, 0x65, 0x5f, 0x70, 0x65, 0x72, + 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x1b, 0x65, + 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x47, 0x61, 0x73, 0x50, 0x72, 0x69, 0x63, 0x65, + 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x61, 0x67, 0x65, 0x12, 0x22, 0x0a, 0x0d, 0x69, 0x6d, + 0x5f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0b, 0x69, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x22, 0xe5, + 0x01, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x47, 0x45, 0x52, 0x12, 0x21, 0x0a, 0x0c, + 0x62, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0b, 0x62, 0x61, 0x74, 0x63, 0x68, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, + 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x28, 0x0a, + 0x10, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x5f, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x72, 0x6f, 0x6f, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x45, + 0x78, 0x69, 0x74, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x69, 0x6e, 0x62, + 0x61, 0x73, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x63, 0x6f, 0x69, 0x6e, 0x62, + 0x61, 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x66, 0x6f, 0x72, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x66, 0x6f, 0x72, 0x6b, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, + 0x63, 0x68, 0x61, 0x69, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x22, 0x51, 0x0a, 0x08, 0x42, 0x6f, 0x6f, 0x6b, 0x4d, 0x61, + 0x72, 0x6b, 0x12, 0x2f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x1b, 0x2e, 0x64, 0x61, 0x74, 0x61, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x31, + 0x2e, 0x42, 0x6f, 0x6f, 0x6b, 0x6d, 0x61, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2a, 0x62, 0x0a, 0x0c, 0x42, 0x6f, 0x6f, + 0x6b, 0x6d, 0x61, 0x72, 0x6b, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x42, 0x4f, 0x4f, + 0x4b, 0x4d, 0x41, 0x52, 0x4b, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, + 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x42, 0x4f, 0x4f, 0x4b, + 0x4d, 0x41, 0x52, 0x4b, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, + 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x42, 0x4f, 0x4f, 0x4b, 0x4d, 0x41, 0x52, 0x4b, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x4c, 0x32, 0x5f, 0x42, 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x02, 0x2a, 0x8d, 0x01, + 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x45, + 0x4e, 0x54, 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, + 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x45, 0x4e, 0x54, 0x52, 0x59, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x01, 0x12, 0x17, 0x0a, + 0x13, 0x45, 0x4e, 0x54, 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4c, 0x32, 0x5f, 0x42, + 0x4c, 0x4f, 0x43, 0x4b, 0x10, 0x02, 0x12, 0x1a, 0x0a, 0x16, 0x45, 0x4e, 0x54, 0x52, 0x59, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x54, 0x52, 0x41, 0x4e, 0x53, 0x41, 0x43, 0x54, 0x49, 0x4f, 0x4e, + 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x45, 0x4e, 0x54, 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x47, 0x45, 0x52, 0x10, 0x04, 0x42, 0x38, 0x5a, + 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x30, 0x78, 0x50, 0x6f, + 0x6c, 0x79, 0x67, 0x6f, 0x6e, 0x48, 0x65, 0x72, 0x6d, 0x65, 0x7a, 0x2f, 0x7a, 0x6b, 0x65, 0x76, + 0x6d, 0x2d, 0x6e, 0x6f, 0x64, 0x65, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x64, 0x61, 0x74, + 0x61, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_datastream_proto_rawDescOnce sync.Once + file_datastream_proto_rawDescData = file_datastream_proto_rawDesc +) + +func file_datastream_proto_rawDescGZIP() []byte { + file_datastream_proto_rawDescOnce.Do(func() { + file_datastream_proto_rawDescData = protoimpl.X.CompressGZIP(file_datastream_proto_rawDescData) + }) + return file_datastream_proto_rawDescData +} + +var file_datastream_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_datastream_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_datastream_proto_goTypes = []interface{}{ + (BookmarkType)(0), // 0: datastream.v1.BookmarkType + (EntryType)(0), // 1: datastream.v1.EntryType + (*Batch)(nil), // 2: datastream.v1.Batch + (*L2Block)(nil), // 3: datastream.v1.L2Block + (*Transaction)(nil), // 4: datastream.v1.Transaction + (*UpdateGER)(nil), // 5: datastream.v1.UpdateGER + (*BookMark)(nil), // 6: datastream.v1.BookMark +} +var file_datastream_proto_depIdxs = []int32{ + 0, // 0: datastream.v1.BookMark.type:type_name -> datastream.v1.BookmarkType + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_datastream_proto_init() } +func file_datastream_proto_init() { + if File_datastream_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_datastream_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Batch); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_datastream_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*L2Block); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_datastream_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Transaction); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_datastream_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*UpdateGER); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_datastream_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BookMark); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_datastream_proto_rawDesc, + NumEnums: 2, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_datastream_proto_goTypes, + DependencyIndexes: file_datastream_proto_depIdxs, + EnumInfos: file_datastream_proto_enumTypes, + MessageInfos: file_datastream_proto_msgTypes, + }.Build() + File_datastream_proto = out.File + file_datastream_proto_rawDesc = nil + file_datastream_proto_goTypes = nil + file_datastream_proto_depIdxs = nil +} diff --git a/state/pgstatestorage/datastream.go b/state/pgstatestorage/datastream.go index 4b15000aeb..31c9814c0a 100644 --- a/state/pgstatestorage/datastream.go +++ b/state/pgstatestorage/datastream.go @@ -78,7 +78,7 @@ func scanL2Block(row pgx.Row) (*state.DSL2Block, error) { } l2Block.GlobalExitRoot = common.HexToHash(gerStr) l2Block.Coinbase = common.HexToAddress(coinbaseStr) - l2Block.Timestamp = timestamp.Unix() + l2Block.Timestamp = uint64(timestamp.Unix()) l2Block.BlockHash = common.HexToHash(blockHashStr) l2Block.StateRoot = common.HexToHash(stateRootStr) @@ -151,9 +151,13 @@ func scanDSL2Transaction(row pgx.Row) (*state.DSL2Transaction, error) { // GetDSBatches returns the DS batches func (p *PostgresStorage) GetDSBatches(ctx context.Context, firstBatchNumber, lastBatchNumber uint64, readWIPBatch bool, dbTx pgx.Tx) ([]*state.DSBatch, error) { var getBatchByNumberSQL = ` - SELECT b.batch_num, b.global_exit_root, b.local_exit_root, b.acc_input_hash, b.state_root, b.timestamp, b.coinbase, b.raw_txs_data, b.forced_batch_num, b.wip, f.fork_id - FROM state.batch b, state.fork_id f - WHERE b.batch_num >= $1 AND b.batch_num <= $2 AND batch_num between f.from_batch_num AND f.to_batch_num` + SELECT b.batch_num, b.global_exit_root, b.local_exit_root, b.acc_input_hash, b.state_root, b.timestamp, b.coinbase, b.raw_txs_data, b.forced_batch_num, b.wip, f.fork_id, vb.timestamp_batch_etrog + FROM state.batch b + LEFT JOIN + state.fork_id f ON b.batch_num BETWEEN f.from_batch_num AND f.to_batch_num + LEFT JOIN + state.virtual_batch vb ON b.batch_num = vb.batch_num + WHERE b.batch_num >= $1 AND b.batch_num <= $2` if !readWIPBatch { getBatchByNumberSQL += " AND b.wip is false" @@ -205,6 +209,7 @@ func scanDSBatch(row pgx.Row) (state.DSBatch, error) { &batch.ForcedBatchNum, &batch.WIP, &batch.ForkID, + &batch.EtrogTimestamp, ) if err != nil { return batch, err diff --git a/state/test/datastream_test.go b/state/test/datastream_test.go deleted file mode 100644 index 8860cf9740..0000000000 --- a/state/test/datastream_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package test - -import ( - "fmt" - "testing" - "time" - - "github.com/0xPolygonHermez/zkevm-node/state" - "github.com/ethereum/go-ethereum/common" - "github.com/stretchr/testify/assert" -) - -func TestL2BlockStartEncode(t *testing.T) { - l2BlockStart := state.DSL2BlockStart{ - BatchNumber: 1, - L2BlockNumber: 2, - Timestamp: 3, - DeltaTimestamp: 4, - L1InfoTreeIndex: 5, - L1BlockHash: common.HexToHash("0x06"), - GlobalExitRoot: common.HexToHash("0x07"), - Coinbase: common.HexToAddress("0x08"), - ForkID: 9, - ChainID: 10, - } - - encoded := l2BlockStart.Encode() - expected := []byte{ - 0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 2, - 0, 0, 0, 0, 0, 0, 0, 3, - 0, 0, 0, 4, - 0, 0, 0, 5, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, - 0, 9, - 0, 0, 0, 10} - - assert.Equal(t, expected, encoded) -} - -func TestL2TransactionEncode(t *testing.T) { - l2Transaction := state.DSL2Transaction{ - EffectiveGasPricePercentage: 128, // 1 byte - IsValid: 1, // 1 byte - StateRoot: common.HexToHash("0x010203"), // 32 bytes - EncodedLength: 5, // 4 bytes - Encoded: []byte{1, 2, 3, 4, 5}, // 5 bytes - } - - encoded := l2Transaction.Encode() - expected := []byte{128, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 5, 1, 2, 3, 4, 5} - assert.Equal(t, expected, encoded) -} - -func TestL2BlockEndEncode(t *testing.T) { - l2BlockEnd := state.DSL2BlockEnd{ - L2BlockNumber: 1, // 8 bytes - BlockHash: common.HexToHash("0x02"), // 32 bytes - StateRoot: common.HexToHash("0x03"), // 32 bytes - } - - encoded := l2BlockEnd.Encode() - expected := []byte{0, 0, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3} - - assert.Equal(t, expected, encoded) -} - -func TestCalculateSCPosition(t *testing.T) { - a := time.Now() - blockNumber := uint64(2934867) - expected := common.HexToHash("0xaa93c484856be45716623765b429a967296594ca362e61e91d671fb422e0f744") - position := state.GetSystemSCPosition(blockNumber) - assert.Equal(t, expected, common.BytesToHash(position)) - b := time.Now() - - c := b.Sub(a) - fmt.Println(c) -} diff --git a/test/docker-compose.yml b/test/docker-compose.yml index feead65a40..fd82ce7b5f 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -513,9 +513,12 @@ services: zkevm-prover: container_name: zkevm-prover image: hermeznetwork/zkevm-prover:v6.0.0 + platform: linux/amd64 ports: - 50061:50061 # MT - 50071:50071 # Executor + environment: + - EXPERIMENTAL_DOCKER_DESKTOP_FORCE_QEMU=1 volumes: - ./config/test.prover.config.json:/usr/src/app/config.json command: > @@ -601,7 +604,10 @@ services: zkevm-permissionless-prover: container_name: zkevm-permissionless-prover + platform: linux/amd64 image: hermeznetwork/zkevm-prover:v6.0.0 + environment: + - EXPERIMENTAL_DOCKER_DESKTOP_FORCE_QEMU=1 ports: # - 50058:50058 # Prover - 50059:50052 # Mock prover diff --git a/tools/datastreamer/Makefile b/tools/datastreamer/Makefile index 5698417c34..e79b0cb1a8 100644 --- a/tools/datastreamer/Makefile +++ b/tools/datastreamer/Makefile @@ -6,12 +6,14 @@ check-go: # Targets that require the checks generate-file: check-go -reprocess: check-go decode-entry-offline: check-go decode-l2block-offline: check-go decode-entry: check-go decode-l2block: check-go +decode-batch: check-go +decode-batch-offline: check-go truncate: check-go +dump-batch: check-go arguments := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) @@ -27,6 +29,10 @@ decode-entry: ## Runs the tool to decode a given entry number decode-l2block: ## Runs the tool to decode a given L2 block go run main.go decode-l2block -cfg config/tool.config.toml -l2block $(arguments) +.PHONY: decode-batch +decode-batch: ## Runs the tool to decode a given batch + go run main.go decode-batch -cfg config/tool.config.toml -batch $(arguments) + .PHONY: decode-entry-offline decode-entry-offline: ## Runs the offline tool to decode a given entry number go run main.go decode-entry-offline -cfg config/tool.config.toml -entry $(arguments) @@ -35,13 +41,21 @@ decode-entry-offline: ## Runs the offline tool to decode a given entry number decode-l2block-offline: ## Runs the offline tool to decode a given L2 block go run main.go decode-l2block-offline -cfg config/tool.config.toml -l2block $(arguments) +.PHONY: decode-batch-offline +decode-batch-offline: ## Runs the offline tool to decode a given batch + go run main.go decode-batch-offline -cfg config/tool.config.toml -batch $(arguments) + .PHONY: truncate truncate: ## Runs the offline tool to truncate the stream file go run main.go truncate -cfg config/tool.config.toml -entry $(arguments) -# .PHONY: reprocess -reprocess: ## Runs the tool to reprocess the information in the stream since a given l2 block - go run main.go reprocess -cfg config/tool.config.toml -genesis ../test/config/test.genesis.config.json -l2block $(arguments) +.PHONY: dump-batch +dump-batch: ## Runs the tool to dump a given batch to file + go run main.go dump-batch -cfg config/tool.config.toml -d -batch $(arguments) + +.PHONY: dump-batch-offline +dump-batch-offline: ## Runs the tool to dump a given batch to file offline + go run main.go dump-batch-offline -cfg config/tool.config.toml -d -batch $(arguments) ## Help display. ## Pulls comments from beside commands and prints a nicely formatted diff --git a/tools/datastreamer/config/tool.config.toml b/tools/datastreamer/config/tool.config.toml index c497f3362f..0e8fc09fc9 100644 --- a/tools/datastreamer/config/tool.config.toml +++ b/tools/datastreamer/config/tool.config.toml @@ -1,11 +1,11 @@ [Online] -URI = "zkevm-sequencer:6900" +URI = "localhost:6900" StreamType = 1 [Offline] Port = 6901 Filename = "datastream.bin" -Version = 1 +Version = 3 ChainID = 1440 UpgradeEtrogBatchNumber = 0 @@ -18,12 +18,8 @@ Port = "5432" EnableLog = false MaxConns = 200 -[Executor] -URI = "zkevm-prover:50071" -MaxGRPCMessageSize = 100000000 - [MerkleTree] -URI = "zkevm-prover:50061" +URI = "localhost:50061" MaxThreads = 20 CacheFile = "merkle_tree_cache.json" diff --git a/tools/datastreamer/main.go b/tools/datastreamer/main.go index 975e4c7ecd..bb84aeebf3 100644 --- a/tools/datastreamer/main.go +++ b/tools/datastreamer/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/binary" "encoding/json" "fmt" "math/big" @@ -12,19 +11,16 @@ import ( "github.com/0xPolygonHermez/zkevm-data-streamer/datastreamer" "github.com/0xPolygonHermez/zkevm-data-streamer/log" - nodeConfig "github.com/0xPolygonHermez/zkevm-node/config" "github.com/0xPolygonHermez/zkevm-node/db" - "github.com/0xPolygonHermez/zkevm-node/encoding" - "github.com/0xPolygonHermez/zkevm-node/hex" "github.com/0xPolygonHermez/zkevm-node/merkletree" "github.com/0xPolygonHermez/zkevm-node/state" + "github.com/0xPolygonHermez/zkevm-node/state/datastream" "github.com/0xPolygonHermez/zkevm-node/state/pgstatestorage" - "github.com/0xPolygonHermez/zkevm-node/state/runtime/executor" "github.com/0xPolygonHermez/zkevm-node/tools/datastreamer/config" "github.com/ethereum/go-ethereum/common" "github.com/fatih/color" - "github.com/google/uuid" "github.com/urfave/cli/v2" + "google.golang.org/protobuf/proto" ) const ( @@ -41,14 +37,6 @@ var ( Required: true, } - genesisFileFlag = cli.StringFlag{ - Name: config.FlagGenesis, - Aliases: []string{"g"}, - Usage: "Genesis `FILE`", - DefaultText: "./config/genesis.json", - Required: true, - } - entryFlag = cli.Uint64Flag{ Name: "entry", Aliases: []string{"e"}, @@ -63,10 +51,17 @@ var ( Required: true, } - updateFileFlag = cli.BoolFlag{ - Name: "update", - Aliases: []string{"u"}, - Usage: "Update `FILE`", + batchFlag = cli.Uint64Flag{ + Name: "batch", + Aliases: []string{"bn"}, + Usage: "Batch `NUMBER`", + Required: true, + } + + dumpFlag = cli.BoolFlag{ + Name: "dump", + Aliases: []string{"d"}, + Usage: "Dump batch to file", Required: false, } ) @@ -86,18 +81,6 @@ func main() { &configFileFlag, }, }, - { - Name: "reprocess", - Aliases: []string{}, - Usage: "Reprocess l2block since a given l2block number", - Action: reprocess, - Flags: []cli.Flag{ - &configFileFlag, - &genesisFileFlag, - &l2blockFlag, - &updateFileFlag, - }, - }, { Name: "decode-entry-offline", Aliases: []string{}, @@ -118,6 +101,16 @@ func main() { &l2blockFlag, }, }, + { + Name: "decode-batch-offline", + Aliases: []string{}, + Usage: "Decodes a batch offline", + Action: decodeBatchOffline, + Flags: []cli.Flag{ + &configFileFlag, + &batchFlag, + }, + }, { Name: "decode-entry", Aliases: []string{}, @@ -138,6 +131,16 @@ func main() { &l2blockFlag, }, }, + { + Name: "decode-batch", + Aliases: []string{}, + Usage: "Decodes a batch", + Action: decodeBatch, + Flags: []cli.Flag{ + &configFileFlag, + &batchFlag, + }, + }, { Name: "truncate", Aliases: []string{}, @@ -148,6 +151,28 @@ func main() { &entryFlag, }, }, + { + Name: "dump-batch", + Aliases: []string{}, + Usage: "Dumps a batch to file", + Action: decodeBatch, + Flags: []cli.Flag{ + &configFileFlag, + &batchFlag, + &dumpFlag, + }, + }, + { + Name: "dump-batch-offline", + Aliases: []string{}, + Usage: "Dumps a batch to file offline", + Action: decodeBatchOffline, + Flags: []cli.Flag{ + &configFileFlag, + &batchFlag, + &dumpFlag, + }, + }, } err := app.Run(os.Args) @@ -272,7 +297,7 @@ func generate(cliCtx *cli.Context) error { } } - err = state.GenerateDataStreamerFile(cliCtx.Context, streamServer, stateDB, false, &imStateRoots, c.Offline.ChainID, c.Offline.UpgradeEtrogBatchNumber) // nolint:gomnd + err = state.GenerateDataStreamFile(cliCtx.Context, streamServer, stateDB, false, &imStateRoots, c.Offline.ChainID, c.Offline.UpgradeEtrogBatchNumber) if err != nil { log.Error(err) os.Exit(1) @@ -299,13 +324,18 @@ func getImStateRoots(ctx context.Context, start, end uint64, isStateRoots *map[u log.Errorf("Error: %v\n", err) os.Exit(1) } + + if common.BytesToHash(imStateRoot.Bytes()) == state.ZeroHash && x != 0 { + break + } + imStateRootMux.Lock() (*isStateRoots)[x] = imStateRoot.Bytes() imStateRootMux.Unlock() } } -func reprocess(cliCtx *cli.Context) error { +func decodeEntry(cliCtx *cli.Context) error { c, err := config.Load(cliCtx) if err != nil { log.Error(err) @@ -314,230 +344,90 @@ func reprocess(cliCtx *cli.Context) error { log.Init(c.Log) - ctx := cliCtx.Context - - genesisFileAsStr, err := nodeConfig.LoadGenesisFileAsString(cliCtx.String(config.FlagGenesis)) + client, err := datastreamer.NewClient(c.Online.URI, c.Online.StreamType) if err != nil { - fmt.Printf("failed to load genesis file. Error: %v", err) + log.Error(err) os.Exit(1) } - networkConfig, err := nodeConfig.LoadGenesisFromJSONString(genesisFileAsStr) + err = client.Start() if err != nil { - fmt.Printf("failed to load genesis configuration from file. Error: %v", err) + log.Error(err) os.Exit(1) } - currentL2BlockNumber := cliCtx.Uint64("l2block") - var stateRoot []byte - - streamServer, err := initializeStreamServer(c) + entry, err := client.ExecCommandGetEntry(cliCtx.Uint64("entry")) if err != nil { log.Error(err) os.Exit(1) } - if currentL2BlockNumber == 0 { - printColored(color.FgHiYellow, "\n\nSetting Genesis block\n\n") - - mtDBServerConfig := merkletree.Config{URI: c.MerkleTree.URI} - var mtDBCancel context.CancelFunc - mtDBServiceClient, mtDBClientConn, mtDBCancel := merkletree.NewMTDBServiceClient(ctx, mtDBServerConfig) - defer func() { - mtDBCancel() - mtDBClientConn.Close() - }() - - stateTree := merkletree.NewStateTree(mtDBServiceClient) - - stateRoot, err = setGenesis(ctx, stateTree, networkConfig.Genesis) - if err != nil { - log.Error(err) - os.Exit(1) - } - - // Get Genesis block from the file and validate the state root - bookMark := state.DSBookMark{ - Type: state.BookMarkTypeL2Block, - Value: 0, - } - - firstEntry, err := streamServer.GetFirstEventAfterBookmark(bookMark.Encode()) - if err != nil { - log.Error(err) - os.Exit(1) - } - printEntry(firstEntry) - - secondEntry, err := streamServer.GetEntry(firstEntry.Number + 1) - if err != nil { - log.Error(err) - os.Exit(1) - } - printEntry(secondEntry) + printEntry(entry) + return nil +} - if common.Bytes2Hex(stateRoot) != common.Bytes2Hex(secondEntry.Data[40:72]) { - printColored(color.FgRed, "\nError: Genesis state root does not match\n\n") - os.Exit(1) - } else { - printColored(color.FgGreen, "\nGenesis state root matches\n\n") - } - currentL2BlockNumber++ +func decodeL2Block(cliCtx *cli.Context) error { + c, err := config.Load(cliCtx) + if err != nil { + log.Error(err) + os.Exit(1) } - // Connect to the executor - executorClient, executorClientConn, executorCancel := executor.NewExecutorClient(ctx, c.Executor) - defer func() { - executorCancel() - executorClientConn.Close() - }() + log.Init(c.Log) - bookMark := state.DSBookMark{ - Type: state.BookMarkTypeL2Block, - Value: currentL2BlockNumber, + client, err := datastreamer.NewClient(c.Online.URI, c.Online.StreamType) + if err != nil { + log.Error(err) + os.Exit(1) } - startEntry, err := streamServer.GetFirstEventAfterBookmark(bookMark.Encode()) + err = client.Start() if err != nil { log.Error(err) os.Exit(1) } - var previousStateRoot = stateRoot - var maxEntry = streamServer.GetHeader().TotalEntries - - for x := startEntry.Number; x < maxEntry; x++ { - printColored(color.FgHiYellow, fmt.Sprintf("\nProcessing entity: %d\n", x)) - - currentEntry, err := streamServer.GetEntry(x) - if err != nil { - log.Error(err) - os.Exit(1) - } - - var processBatchRequest *executor.ProcessBatchRequest - var expectedNewRoot []byte - var entryToUpdate *datastreamer.FileEntry - - switch currentEntry.Type { - case state.EntryTypeBookMark: - printEntry(currentEntry) - entryToUpdate = nil - continue - case state.EntryTypeUpdateGER: - printEntry(currentEntry) - processBatchRequest = &executor.ProcessBatchRequest{ - OldBatchNum: binary.BigEndian.Uint64(currentEntry.Data[0:8]) - 1, - Coinbase: common.Bytes2Hex(currentEntry.Data[48:68]), - BatchL2Data: nil, - OldStateRoot: previousStateRoot, - GlobalExitRoot: currentEntry.Data[16:48], - OldAccInputHash: []byte{}, - EthTimestamp: binary.BigEndian.Uint64(currentEntry.Data[8:16]), - UpdateMerkleTree: uint32(1), - ChainId: c.Offline.ChainID, - ForkId: uint64(binary.BigEndian.Uint16(currentEntry.Data[68:70])), - } - - expectedNewRoot = currentEntry.Data[70:102] - entryToUpdate = nil - case state.EntryTypeL2BlockStart: - startEntry = currentEntry - printEntry(startEntry) - - txEntry, err := streamServer.GetEntry(startEntry.Number + 1) - if err != nil { - log.Error(err) - os.Exit(1) - } - printEntry(txEntry) - - endEntry, err := streamServer.GetEntry(startEntry.Number + 2) //nolint:gomnd - if err != nil { - log.Error(err) - os.Exit(1) - } - printEntry(endEntry) - - forkID := uint64(binary.BigEndian.Uint16(startEntry.Data[76:78])) - - tx, err := state.DecodeTx(common.Bytes2Hex((txEntry.Data[6:]))) - if err != nil { - log.Error(err) - os.Exit(1) - } + l2BlockNumber := cliCtx.Uint64("l2block") - // Get the old state root - oldStateRoot := getOldStateRoot(startEntry.Number, streamServer) + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK, + Value: l2BlockNumber, + } - // RLP encode the transaction using the proper fork id - batchL2Data, err := state.EncodeTransaction(*tx, txEntry.Data[0], forkID) //nolint:gomnd - if err != nil { - log.Error(err) - os.Exit(1) - } + marshalledBookMark, err := proto.Marshal(bookMark) + if err != nil { + return err + } - processBatchRequest = &executor.ProcessBatchRequest{ - OldBatchNum: binary.BigEndian.Uint64(startEntry.Data[0:8]) - 1, - Coinbase: common.Bytes2Hex(startEntry.Data[56:76]), - BatchL2Data: batchL2Data, - OldStateRoot: oldStateRoot, - GlobalExitRoot: startEntry.Data[24:56], - OldAccInputHash: []byte{}, - EthTimestamp: binary.BigEndian.Uint64(startEntry.Data[16:24]), - UpdateMerkleTree: uint32(1), - ChainId: c.Offline.ChainID, - ForkId: uint64(binary.BigEndian.Uint16(startEntry.Data[76:78])), - } + firstEntry, err := client.ExecCommandGetBookmark(marshalledBookMark) + if err != nil { + log.Error(err) + os.Exit(1) + } + printEntry(firstEntry) - expectedNewRoot = endEntry.Data[40:72] - entryToUpdate = &endEntry - x += 2 //nolint:gomnd - } + secondEntry, err := client.ExecCommandGetEntry(firstEntry.Number + 1) + if err != nil { + log.Error(err) + os.Exit(1) + } - // Process batch - processBatchResponse, err := executorClient.ProcessBatch(ctx, processBatchRequest) + i := uint64(2) //nolint:gomnd + for secondEntry.Type == datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_TRANSACTION) { + printEntry(secondEntry) + entry, err := client.ExecCommandGetEntry(firstEntry.Number + i) if err != nil { log.Error(err) os.Exit(1) } - - if processBatchResponse.Error != executor.ExecutorError_EXECUTOR_ERROR_NO_ERROR { - fmt.Printf("Error: %v\n", processBatchResponse.Error) - os.Exit(1) - } - - if common.Bytes2Hex(processBatchResponse.NewStateRoot) != common.Bytes2Hex(expectedNewRoot) { - printColored(color.FgRed, "\nNew state root does not match\n\n") - printColored(color.FgRed, fmt.Sprintf("Old State Root.........: %s\n", "0x"+common.Bytes2Hex(processBatchRequest.GetOldStateRoot()))) - printColored(color.FgRed, fmt.Sprintf("New State Root.........: %s\n", "0x"+common.Bytes2Hex(processBatchResponse.NewStateRoot))) - printColored(color.FgRed, fmt.Sprintf("Expected New State Root: %s\n", "0x"+common.Bytes2Hex(expectedNewRoot))) - // Check if we must update the file with the new state root - if cliCtx.Bool("update") { - if entryToUpdate.Type != state.EntryTypeL2BlockEnd { - printColored(color.FgRed, "Error: Entry to update is not a L2BlockEnd\n") - os.Exit(1) - } - blockEnd := state.DSL2BlockEnd{}.Decode(entryToUpdate.Data) - blockEnd.StateRoot = common.BytesToHash(processBatchResponse.NewStateRoot) - err = streamServer.UpdateEntryData(entryToUpdate.Number, state.EntryTypeL2BlockEnd, blockEnd.Encode()) - if err != nil { - printColored(color.FgRed, fmt.Sprintf("Error: %v\n", err)) - os.Exit(1) - } - } else { - break - } - } else { - printColored(color.FgGreen, "New state root matches\n") - previousStateRoot = processBatchResponse.NewStateRoot - } + secondEntry = entry + i++ } return nil } -func decodeEntry(cliCtx *cli.Context) error { +func decodeEntryOffline(cliCtx *cli.Context) error { c, err := config.Load(cliCtx) if err != nil { log.Error(err) @@ -546,29 +436,24 @@ func decodeEntry(cliCtx *cli.Context) error { log.Init(c.Log) - client, err := datastreamer.NewClient(c.Online.URI, c.Online.StreamType) - if err != nil { - log.Error(err) - os.Exit(1) - } - - err = client.Start() + streamServer, err := initializeStreamServer(c) if err != nil { log.Error(err) os.Exit(1) } - entry, err := client.ExecCommandGetEntry(cliCtx.Uint64("entry")) + entry, err := streamServer.GetEntry(cliCtx.Uint64("entry")) if err != nil { log.Error(err) os.Exit(1) } printEntry(entry) + return nil } -func decodeL2Block(cliCtx *cli.Context) error { +func decodeL2BlockOffline(cliCtx *cli.Context) error { c, err := config.Load(cliCtx) if err != nil { log.Error(err) @@ -577,13 +462,7 @@ func decodeL2Block(cliCtx *cli.Context) error { log.Init(c.Log) - client, err := datastreamer.NewClient(c.Online.URI, c.Online.StreamType) - if err != nil { - log.Error(err) - os.Exit(1) - } - - err = client.Start() + streamServer, err := initializeStreamServer(c) if err != nil { log.Error(err) os.Exit(1) @@ -591,41 +470,45 @@ func decodeL2Block(cliCtx *cli.Context) error { l2BlockNumber := cliCtx.Uint64("l2block") - bookMark := state.DSBookMark{ - Type: state.BookMarkTypeL2Block, + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK, Value: l2BlockNumber, } - firstEntry, err := client.ExecCommandGetBookmark(bookMark.Encode()) + marshalledBookMark, err := proto.Marshal(bookMark) + if err != nil { + return err + } + + firstEntry, err := streamServer.GetFirstEventAfterBookmark(marshalledBookMark) if err != nil { log.Error(err) os.Exit(1) } printEntry(firstEntry) - secondEntry, err := client.ExecCommandGetEntry(firstEntry.Number + 1) + secondEntry, err := streamServer.GetEntry(firstEntry.Number + 1) if err != nil { log.Error(err) os.Exit(1) } - printEntry(secondEntry) i := uint64(2) //nolint:gomnd - for secondEntry.Type == state.EntryTypeL2Tx { - entry, err := client.ExecCommandGetEntry(firstEntry.Number + i) + + for secondEntry.Type == datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_TRANSACTION) { + printEntry(secondEntry) + secondEntry, err = streamServer.GetEntry(firstEntry.Number + i) if err != nil { log.Error(err) os.Exit(1) } - secondEntry = entry - printEntry(secondEntry) i++ } return nil } -func decodeEntryOffline(cliCtx *cli.Context) error { +func truncate(cliCtx *cli.Context) error { c, err := config.Load(cliCtx) if err != nil { log.Error(err) @@ -640,18 +523,19 @@ func decodeEntryOffline(cliCtx *cli.Context) error { os.Exit(1) } - entry, err := streamServer.GetEntry(cliCtx.Uint64("entry")) + err = streamServer.TruncateFile(cliCtx.Uint64("entry")) if err != nil { log.Error(err) os.Exit(1) } - printEntry(entry) + printColored(color.FgGreen, "File truncated\n") return nil } -func decodeL2BlockOffline(cliCtx *cli.Context) error { +func decodeBatch(cliCtx *cli.Context) error { + var batchData = []byte{} c, err := config.Load(cliCtx) if err != nil { log.Error(err) @@ -660,48 +544,85 @@ func decodeL2BlockOffline(cliCtx *cli.Context) error { log.Init(c.Log) - streamServer, err := initializeStreamServer(c) + client, err := datastreamer.NewClient(c.Online.URI, c.Online.StreamType) if err != nil { log.Error(err) os.Exit(1) } - l2BlockNumber := cliCtx.Uint64("l2block") + err = client.Start() + if err != nil { + log.Error(err) + os.Exit(1) + } - bookMark := state.DSBookMark{ - Type: state.BookMarkTypeL2Block, - Value: l2BlockNumber, + batchNumber := cliCtx.Uint64("batch") + + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_BATCH, + Value: batchNumber, } - firstEntry, err := streamServer.GetFirstEventAfterBookmark(bookMark.Encode()) + marshalledBookMark, err := proto.Marshal(bookMark) + if err != nil { + return err + } + + firstEntry, err := client.ExecCommandGetBookmark(marshalledBookMark) if err != nil { log.Error(err) os.Exit(1) } printEntry(firstEntry) - secondEntry, err := streamServer.GetEntry(firstEntry.Number + 1) + batchData = append(batchData, firstEntry.Encode()...) + + secondEntry, err := client.ExecCommandGetEntry(firstEntry.Number + 1) if err != nil { log.Error(err) os.Exit(1) } + printEntry(secondEntry) + + batchData = append(batchData, secondEntry.Encode()...) i := uint64(2) //nolint:gomnd - printEntry(secondEntry) - for secondEntry.Type == state.EntryTypeL2Tx { - secondEntry, err = streamServer.GetEntry(firstEntry.Number + i) + for { + entry, err := client.ExecCommandGetEntry(firstEntry.Number + i) if err != nil { log.Error(err) os.Exit(1) } + + if entry.Type == state.EntryTypeBookMark { + if err := proto.Unmarshal(entry.Data, bookMark); err != nil { + return err + } + if bookMark.Type == datastream.BookmarkType_BOOKMARK_TYPE_BATCH { + break + } + } + + secondEntry = entry printEntry(secondEntry) + batchData = append(batchData, secondEntry.Encode()...) i++ } + // Dump batchdata to a file + if cliCtx.Bool("dump") { + err = os.WriteFile(fmt.Sprintf("batch_%d.bin", batchNumber), batchData, 0644) // nolint:gosec, gomnd + if err != nil { + log.Error(err) + os.Exit(1) + } + } + return nil } -func truncate(cliCtx *cli.Context) error { +func decodeBatchOffline(cliCtx *cli.Context) error { + var batchData = []byte{} c, err := config.Load(cliCtx) if err != nil { log.Error(err) @@ -716,26 +637,84 @@ func truncate(cliCtx *cli.Context) error { os.Exit(1) } - err = streamServer.TruncateFile(cliCtx.Uint64("entry")) + batchNumber := cliCtx.Uint64("batch") + + bookMark := &datastream.BookMark{ + Type: datastream.BookmarkType_BOOKMARK_TYPE_BATCH, + Value: batchNumber, + } + + marshalledBookMark, err := proto.Marshal(bookMark) + if err != nil { + return err + } + + firstEntry, err := streamServer.GetFirstEventAfterBookmark(marshalledBookMark) if err != nil { log.Error(err) os.Exit(1) } + printEntry(firstEntry) + batchData = append(batchData, firstEntry.Encode()...) - printColored(color.FgGreen, "File truncated\n") + secondEntry, err := streamServer.GetEntry(firstEntry.Number + 1) + if err != nil { + log.Error(err) + os.Exit(1) + } + + i := uint64(2) //nolint:gomnd + printEntry(secondEntry) + batchData = append(batchData, secondEntry.Encode()...) + for { + secondEntry, err = streamServer.GetEntry(firstEntry.Number + i) + if err != nil { + log.Error(err) + os.Exit(1) + } + + if secondEntry.Type == state.EntryTypeBookMark { + if err := proto.Unmarshal(secondEntry.Data, bookMark); err != nil { + return err + } + if bookMark.Type == datastream.BookmarkType_BOOKMARK_TYPE_BATCH { + break + } + } + + printEntry(secondEntry) + batchData = append(batchData, secondEntry.Encode()...) + i++ + } + + // Dump batchdata to a file + if cliCtx.Bool("dump") { + err = os.WriteFile(fmt.Sprintf("offline_batch_%d.bin", batchNumber), batchData, 0644) // nolint:gosec, gomnd + if err != nil { + log.Error(err) + os.Exit(1) + } + } return nil } func printEntry(entry datastreamer.FileEntry) { - var bookmarkTypeDesc = map[byte]string{ - state.BookMarkTypeL2Block: "L2 Block Number", - state.BookMarkTypeBatch: "Batch Number", + var bookmarkTypeDesc = map[datastream.BookmarkType]string{ + datastream.BookmarkType_BOOKMARK_TYPE_UNSPECIFIED: "Unspecified", + datastream.BookmarkType_BOOKMARK_TYPE_BATCH: "Batch Number", + datastream.BookmarkType_BOOKMARK_TYPE_L2_BLOCK: "L2 Block Number", } switch entry.Type { case state.EntryTypeBookMark: - bookmark := state.DSBookMark{}.Decode(entry.Data) + bookmark := &datastream.BookMark{} + err := proto.Unmarshal(entry.Data, bookmark) + if err != nil { + log.Error(err) + os.Exit(1) + } + printColored(color.FgGreen, "Entry Type......: ") printColored(color.FgHiYellow, "BookMark\n") printColored(color.FgGreen, "Entry Number....: ") @@ -744,48 +723,83 @@ func printEntry(entry datastreamer.FileEntry) { printColored(color.FgHiWhite, fmt.Sprintf("%d (%s)\n", bookmark.Type, bookmarkTypeDesc[bookmark.Type])) printColored(color.FgGreen, "Value...........: ") printColored(color.FgHiWhite, fmt.Sprintf("%d\n", bookmark.Value)) - case state.EntryTypeL2BlockStart: - blockStart := state.DSL2BlockStart{}.Decode(entry.Data) + case datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_L2_BLOCK): + l2Block := &datastream.L2Block{} + err := proto.Unmarshal(entry.Data, l2Block) + if err != nil { + log.Error(err) + os.Exit(1) + } + printColored(color.FgGreen, "Entry Type......: ") - printColored(color.FgHiYellow, "L2 Block Start\n") + printColored(color.FgHiYellow, "L2 Block\n") printColored(color.FgGreen, "Entry Number....: ") printColored(color.FgHiWhite, fmt.Sprintf("%d\n", entry.Number)) - printColored(color.FgGreen, "Batch Number....: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", blockStart.BatchNumber)) printColored(color.FgGreen, "L2 Block Number.: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", blockStart.L2BlockNumber)) + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", l2Block.Number)) + printColored(color.FgGreen, "Batch Number....: ") + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", l2Block.BatchNumber)) printColored(color.FgGreen, "Timestamp.......: ") - printColored(color.FgHiWhite, fmt.Sprintf("%v (%d)\n", time.Unix(blockStart.Timestamp, 0), blockStart.Timestamp)) + printColored(color.FgHiWhite, fmt.Sprintf("%d (%v)\n", l2Block.Timestamp, time.Unix(int64(l2Block.Timestamp), 0))) printColored(color.FgGreen, "Delta Timestamp.: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", blockStart.DeltaTimestamp)) - printColored(color.FgGreen, "L1 InfoTree Idx.: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", blockStart.L1InfoTreeIndex)) + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", l2Block.DeltaTimestamp)) + printColored(color.FgGreen, "Min. Timestamp..: ") + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", l2Block.MinTimestamp)) printColored(color.FgGreen, "L1 Block Hash...: ") - printColored(color.FgHiWhite, fmt.Sprintf("%s\n", blockStart.L1BlockHash)) + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", common.BytesToHash(l2Block.L1Blockhash))) + printColored(color.FgGreen, "L1 InfoTree Idx.: ") + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", l2Block.L1InfotreeIndex)) + printColored(color.FgGreen, "Block Hash......: ") + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", common.BytesToHash(l2Block.Hash))) + printColored(color.FgGreen, "State Root......: ") + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", common.BytesToHash(l2Block.StateRoot))) printColored(color.FgGreen, "Global Exit Root: ") - printColored(color.FgHiWhite, fmt.Sprintf("%s\n", blockStart.GlobalExitRoot)) + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", common.BytesToHash(l2Block.GlobalExitRoot))) printColored(color.FgGreen, "Coinbase........: ") - printColored(color.FgHiWhite, fmt.Sprintf("%s\n", blockStart.Coinbase)) + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", common.BytesToAddress(l2Block.Coinbase))) + case datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_BATCH): + batch := &datastream.Batch{} + err := proto.Unmarshal(entry.Data, batch) + if err != nil { + log.Error(err) + os.Exit(1) + } + printColored(color.FgGreen, "Entry Type......: ") + printColored(color.FgHiYellow, "Batch\n") + printColored(color.FgGreen, "Entry Number....: ") + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", entry.Number)) + printColored(color.FgGreen, "Batch Number....: ") + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", batch.Number)) + printColored(color.FgGreen, "State Root......: ") + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", "0x"+common.Bytes2Hex(batch.StateRoot))) + printColored(color.FgGreen, "Local Exit Root.: ") + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", "0x"+common.Bytes2Hex(batch.LocalExitRoot))) printColored(color.FgGreen, "Fork ID.........: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", blockStart.ForkID)) + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", batch.ForkId)) printColored(color.FgGreen, "Chain ID........: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", blockStart.ChainID)) - case state.EntryTypeL2Tx: - dsTx := state.DSL2Transaction{}.Decode(entry.Data) + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", batch.ChainId)) + case datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_TRANSACTION): + dsTx := &datastream.Transaction{} + err := proto.Unmarshal(entry.Data, dsTx) + if err != nil { + log.Error(err) + os.Exit(1) + } + printColored(color.FgGreen, "Entry Type......: ") printColored(color.FgHiYellow, "L2 Transaction\n") printColored(color.FgGreen, "Entry Number....: ") printColored(color.FgHiWhite, fmt.Sprintf("%d\n", entry.Number)) + printColored(color.FgGreen, "L2 Block Number.: ") + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", dsTx.L2BlockNumber)) + printColored(color.FgGreen, "Is Valid........: ") + printColored(color.FgHiWhite, fmt.Sprintf("%t\n", dsTx.IsValid)) + printColored(color.FgGreen, "Data............: ") + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", "0x"+common.Bytes2Hex(dsTx.Encoded))) printColored(color.FgGreen, "Effec. Gas Price: ") printColored(color.FgHiWhite, fmt.Sprintf("%d\n", dsTx.EffectiveGasPricePercentage)) - printColored(color.FgGreen, "Is Valid........: ") - printColored(color.FgHiWhite, fmt.Sprintf("%t\n", dsTx.IsValid == 1)) printColored(color.FgGreen, "IM State Root...: ") - printColored(color.FgHiWhite, fmt.Sprint(dsTx.StateRoot.Hex()+"\n")) - printColored(color.FgGreen, "Encoded Length..: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", dsTx.EncodedLength)) - printColored(color.FgGreen, "Encoded.........: ") - printColored(color.FgHiWhite, fmt.Sprintf("%s\n", "0x"+common.Bytes2Hex(dsTx.Encoded))) + printColored(color.FgHiWhite, fmt.Sprint("0x"+common.Bytes2Hex(dsTx.ImStateRoot)+"\n")) tx, err := state.DecodeTx(common.Bytes2Hex(dsTx.Encoded)) if err != nil { @@ -804,20 +818,14 @@ func printEntry(entry datastreamer.FileEntry) { nonce := tx.Nonce() printColored(color.FgGreen, "Nonce...........: ") printColored(color.FgHiWhite, fmt.Sprintf("%d\n", nonce)) - case state.EntryTypeL2BlockEnd: - blockEnd := state.DSL2BlockEnd{}.Decode(entry.Data) - printColored(color.FgGreen, "Entry Type......: ") - printColored(color.FgHiYellow, "L2 Block End\n") - printColored(color.FgGreen, "Entry Number....: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", entry.Number)) - printColored(color.FgGreen, "L2 Block Number.: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", blockEnd.L2BlockNumber)) - printColored(color.FgGreen, "L2 Block Hash...: ") - printColored(color.FgHiWhite, fmt.Sprint(blockEnd.BlockHash.Hex()+"\n")) - printColored(color.FgGreen, "State Root......: ") - printColored(color.FgHiWhite, fmt.Sprint(blockEnd.StateRoot.Hex()+"\n")) - case state.EntryTypeUpdateGER: - updateGer := state.DSUpdateGER{}.Decode(entry.Data) + case datastreamer.EntryType(datastream.EntryType_ENTRY_TYPE_UPDATE_GER): + updateGer := &datastream.UpdateGER{} + err := proto.Unmarshal(entry.Data, updateGer) + if err != nil { + log.Error(err) + os.Exit(1) + } + printColored(color.FgGreen, "Entry Type......: ") printColored(color.FgHiYellow, "Update GER\n") printColored(color.FgGreen, "Entry Number....: ") @@ -825,17 +833,17 @@ func printEntry(entry datastreamer.FileEntry) { printColored(color.FgGreen, "Batch Number....: ") printColored(color.FgHiWhite, fmt.Sprintf("%d\n", updateGer.BatchNumber)) printColored(color.FgGreen, "Timestamp.......: ") - printColored(color.FgHiWhite, fmt.Sprintf("%v (%d)\n", time.Unix(updateGer.Timestamp, 0), updateGer.Timestamp)) + printColored(color.FgHiWhite, fmt.Sprintf("%v (%d)\n", time.Unix(int64(updateGer.Timestamp), 0), updateGer.Timestamp)) printColored(color.FgGreen, "Global Exit Root: ") - printColored(color.FgHiWhite, fmt.Sprintf("%s\n", updateGer.GlobalExitRoot)) + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", common.Bytes2Hex(updateGer.GlobalExitRoot))) printColored(color.FgGreen, "Coinbase........: ") - printColored(color.FgHiWhite, fmt.Sprintf("%s\n", updateGer.Coinbase)) + printColored(color.FgHiWhite, fmt.Sprintf("%s\n", common.BytesToAddress(updateGer.Coinbase))) printColored(color.FgGreen, "Fork ID.........: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", updateGer.ForkID)) + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", updateGer.ForkId)) printColored(color.FgGreen, "Chain ID........: ") - printColored(color.FgHiWhite, fmt.Sprintf("%d\n", updateGer.ChainID)) + printColored(color.FgHiWhite, fmt.Sprintf("%d\n", updateGer.ChainId)) printColored(color.FgGreen, "State Root......: ") - printColored(color.FgHiWhite, fmt.Sprint(updateGer.StateRoot.Hex()+"\n")) + printColored(color.FgHiWhite, fmt.Sprint(common.Bytes2Hex(updateGer.StateRoot)+"\n")) } } @@ -843,112 +851,3 @@ func printColored(color color.Attribute, text string) { colored := fmt.Sprintf("\x1b[%dm%s\x1b[0m", color, text) fmt.Print(colored) } - -// setGenesis populates state with genesis information -func setGenesis(ctx context.Context, tree *merkletree.StateTree, genesis state.Genesis) ([]byte, error) { - var ( - root common.Hash - newRoot []byte - err error - ) - - if tree == nil { - return newRoot, fmt.Errorf("state tree is nil") - } - - uuid := uuid.New().String() - - for _, action := range genesis.Actions { - address := common.HexToAddress(action.Address) - switch action.Type { - case int(merkletree.LeafTypeBalance): - balance, err := encoding.DecodeBigIntHexOrDecimal(action.Value) - if err != nil { - return newRoot, err - } - newRoot, _, err = tree.SetBalance(ctx, address, balance, newRoot, uuid) - if err != nil { - return newRoot, err - } - case int(merkletree.LeafTypeNonce): - nonce, err := encoding.DecodeBigIntHexOrDecimal(action.Value) - if err != nil { - return newRoot, err - } - newRoot, _, err = tree.SetNonce(ctx, address, nonce, newRoot, uuid) - if err != nil { - return newRoot, err - } - case int(merkletree.LeafTypeCode): - code, err := hex.DecodeHex(action.Bytecode) - if err != nil { - return newRoot, fmt.Errorf("could not decode SC bytecode for address %q: %v", address, err) - } - newRoot, _, err = tree.SetCode(ctx, address, code, newRoot, uuid) - if err != nil { - return newRoot, err - } - case int(merkletree.LeafTypeStorage): - // Parse position and value - positionBI, err := encoding.DecodeBigIntHexOrDecimal(action.StoragePosition) - if err != nil { - return newRoot, err - } - valueBI, err := encoding.DecodeBigIntHexOrDecimal(action.Value) - if err != nil { - return newRoot, err - } - // Store - newRoot, _, err = tree.SetStorageAt(ctx, address, positionBI, valueBI, newRoot, uuid) - if err != nil { - return newRoot, err - } - default: - return newRoot, fmt.Errorf("unknown genesis action type %q", action.Type) - } - } - - root.SetBytes(newRoot) - - // flush state db - err = tree.Flush(ctx, root, uuid) - if err != nil { - fmt.Printf("error flushing state tree after genesis: %v", err) - return newRoot, err - } - - return newRoot, nil -} - -func getOldStateRoot(entityNumber uint64, streamServer *datastreamer.StreamServer) []byte { - var found = false - var entry datastreamer.FileEntry - var err error - - for !found && entityNumber > 1 { - entityNumber-- - entry, err = streamServer.GetEntry(entityNumber) - if err != nil { - log.Error(err) - os.Exit(1) - } - - if entry.Type == state.EntryTypeL2BlockEnd || entry.Type == state.EntryTypeUpdateGER { - found = true - } - } - - if !found { - fmt.Printf("Error: Could not find old state root") - os.Exit(1) - } - - printColored(color.FgHiYellow, "Getting Old State Root from\n") - printEntry(entry) - - if entry.Type == state.EntryTypeUpdateGER { - return entry.Data[70:102] - } - - return entry.Data[40:72] -}