From 9e4302c586288f2a2c78a48e463a6769bd5b87f0 Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Fri, 28 Apr 2023 23:05:57 -0400 Subject: [PATCH 01/11] add block index by height to platformvm --- vms/platformvm/state/mock_state.go | 15 ++++ vms/platformvm/state/state.go | 125 +++++++++++++++++++++++++++-- 2 files changed, 134 insertions(+), 6 deletions(-) diff --git a/vms/platformvm/state/mock_state.go b/vms/platformvm/state/mock_state.go index cdcc2d70d93..01a5b602a45 100644 --- a/vms/platformvm/state/mock_state.go +++ b/vms/platformvm/state/mock_state.go @@ -245,6 +245,21 @@ func (mr *MockStateMockRecorder) DeleteUTXO(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteUTXO", reflect.TypeOf((*MockState)(nil).DeleteUTXO), arg0) } +// GetBlockID mocks base method. +func (m *MockState) GetBlockID(arg0 uint64) (ids.ID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBlockID", arg0) + ret0, _ := ret[0].(ids.ID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBlockID indicates an expected call of GetBlockID. +func (mr *MockStateMockRecorder) GetBlockID(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBlockID", reflect.TypeOf((*MockState)(nil).GetBlockID), arg0) +} + // GetChains mocks base method. func (m *MockState) GetChains(arg0 ids.ID) ([]*txs.Tx, error) { m.ctrl.T.Helper() diff --git a/vms/platformvm/state/state.go b/vms/platformvm/state/state.go index ef604688e5f..48b84c90b96 100644 --- a/vms/platformvm/state/state.go +++ b/vms/platformvm/state/state.go @@ -41,6 +41,7 @@ import ( const ( validatorDiffsCacheSize = 2048 + blockIDCacheSize = 2048 blockCacheSize = 2048 txCacheSize = 2048 rewardUTXOsCacheSize = 2048 @@ -56,6 +57,7 @@ var ( errValidatorSetAlreadyPopulated = errors.New("validator set already populated") errDuplicateValidatorSet = errors.New("duplicate validator set") + blockIDPrefix = []byte("blockID") blockPrefix = []byte("block") validatorsPrefix = []byte("validators") currentPrefix = []byte("current") @@ -122,6 +124,8 @@ type State interface { GetStatelessBlock(blockID ids.ID) (blocks.Block, choices.Status, error) AddStatelessBlock(block blocks.Block, status choices.Status) + GetBlockID(height uint64) (ids.ID, error) + // ValidatorSet adds all the validators and delegators of [subnetID] into // [vdrs]. ValidatorSet(subnetID ids.ID, vdrs validators.Set) error @@ -190,6 +194,8 @@ type stateBlk struct { * | '-. height * | '-. list * | '-- nodeID -> public key + * |-. blockIDs + * | '-- height -> blockID * |-. blocks * | '-- blockID -> block bytes * |-. txs @@ -229,11 +235,13 @@ type state struct { currentHeight uint64 - addedBlocks map[ids.ID]stateBlk // map of blockID -> Block - // cache of blockID -> Block - // If the block isn't known, nil is cached. - blockCache cache.Cacher[ids.ID, *stateBlk] - blockDB database.Database + addedBlockIDs map[uint64]ids.ID // map of height -> blockID + blockIDCache cache.Cacher[uint64, ids.ID] // cache of height -> blockID. If the entry is ids.Empty, it is not in the database + blockIDDB database.Database + + addedBlocks map[ids.ID]stateBlk // map of blockID -> Block + blockCache cache.Cacher[ids.ID, *stateBlk] // cache of blockID -> Block. If the entry is nil, it is not in the database. + blockDB database.Database validatorsDB database.Database currentValidatorsDB database.Database @@ -377,6 +385,15 @@ func new( rewards reward.Calculator, bootstrapped *utils.Atomic[bool], ) (*state, error) { + blockIDCache, err := metercacher.New[uint64, ids.ID]( + "block_id_cache", + metricsReg, + &cache.LRU[uint64, ids.ID]{Size: blockIDCacheSize}, + ) + if err != nil { + return nil, err + } + blockCache, err := metercacher.New[ids.ID, *stateBlk]( "block_cache", metricsReg, @@ -495,6 +512,10 @@ func new( bootstrapped: bootstrapped, baseDB: baseDB, + addedBlockIDs: make(map[uint64]ids.ID), + blockIDCache: blockIDCache, + blockIDDB: prefixdb.New(blockIDPrefix, baseDB), + addedBlocks: make(map[ids.ID]stateBlk), blockCache: blockCache, blockDB: prefixdb.New(blockPrefix, baseDB), @@ -1070,6 +1091,7 @@ func (s *state) load() error { s.loadCurrentValidators(), s.loadPendingValidators(), s.initValidatorSets(), + s.populateBlockHeightIndex(), // Must be called after loadMetadata ) return errs.Err } @@ -1365,6 +1387,7 @@ func (s *state) initValidatorSets() error { func (s *state) write(updateValidators bool, height uint64) error { errs := wrappers.Errs{} errs.Add( + s.writeBlockIDs(), s.writeBlocks(), s.writeCurrentStakers(updateValidators, height), s.writePendingStakers(), @@ -1404,6 +1427,7 @@ func (s *state) Close() error { s.chainDB.Close(), s.singletonDB.Close(), s.blockDB.Close(), + s.blockIDDB.Close(), ) return errs.Err } @@ -1463,7 +1487,9 @@ func (s *state) init(genesisBytes []byte) error { } func (s *state) AddStatelessBlock(block blocks.Block, status choices.Status) { - s.addedBlocks[block.ID()] = stateBlk{ + blkID := block.ID() + s.addedBlockIDs[block.Height()] = blkID + s.addedBlocks[blkID] = stateBlk{ Blk: block, Bytes: block.Bytes(), Status: status, @@ -1496,6 +1522,19 @@ func (s *state) CommitBatch() (database.Batch, error) { return s.baseDB.CommitBatch() } +func (s *state) writeBlockIDs() error { + for height, blkID := range s.addedBlockIDs { + heightKey := database.PackUInt64(height) + + delete(s.addedBlockIDs, height) + s.blockIDCache.Put(height, blkID) + if err := database.PutID(s.blockIDDB, heightKey, blkID); err != nil { + return fmt.Errorf("failed to add blockID: %w", err) + } + } + return nil +} + func (s *state) writeBlocks() error { for blkID, stateBlk := range s.addedBlocks { var ( @@ -1552,6 +1591,33 @@ func (s *state) GetStatelessBlock(blockID ids.ID) (blocks.Block, choices.Status, return blkState.Blk, blkState.Status, nil } +func (s *state) GetBlockID(height uint64) (ids.ID, error) { + if blkID, exists := s.addedBlockIDs[height]; exists { + return blkID, nil + } + if blkID, cached := s.blockIDCache.Get(height); cached { + if blkID == ids.Empty { + return ids.Empty, database.ErrNotFound + } + + return blkID, nil + } + + heightKey := database.PackUInt64(height) + + blkID, err := database.GetID(s.blockIDDB, heightKey) + if err == database.ErrNotFound { + s.blockIDCache.Put(height, ids.Empty) + return ids.Empty, database.ErrNotFound + } + if err != nil { + return ids.Empty, err + } + + s.blockIDCache.Put(height, blkID) + return blkID, nil +} + func (s *state) writeCurrentStakers(updateValidators bool, height uint64) error { heightBytes := database.PackUInt64(height) rawPublicKeyDiffDB := prefixdb.New(heightBytes, s.validatorPublicKeyDiffsDB) @@ -1944,3 +2010,50 @@ func (s *state) writeMetadata() error { } return nil } + +// TODO: Remove after next hard-fork when everybody has run this. +func (s *state) populateBlockHeightIndex() error { + blk, _, err := s.GetStatelessBlock(s.lastAccepted) + if err != nil { + return fmt.Errorf("failed to retrieve last accepted block from local disk: %w", err) + } + + heightKey := database.PackUInt64(blk.Height()) + if _, err := database.GetID(s.blockIDDB, heightKey); err == nil { + // If the last accepted block is in [s.blockIDDB], no backfilling is required. + return nil + } + + blockIterator := s.blockDB.NewIterator() + defer blockIterator.Release() + for blockIterator.Next() { + blkBytes := blockIterator.Value() + + // Note: stored blocks are verified, so it's safe to unmarshal them with GenesisCodec + blkState := stateBlk{} + if _, err := blocks.GenesisCodec.Unmarshal(blkBytes, &blkState); err != nil { + return err + } + + blkState.Blk, err = blocks.Parse(blocks.GenesisCodec, blkState.Bytes) + if err != nil { + return err + } + + blkHeight := blkState.Blk.Height() + blkID := blkState.Blk.ID() + + // Populate the in-memory cache with the [blockIDCacheSize] most recent blocks + if math.AbsDiff(s.currentHeight, blkHeight) <= blockIDCacheSize { + s.addedBlockIDs[blkHeight] = blkID + s.blockIDCache.Put(blkHeight, blkID) + } + + heightKey := database.PackUInt64(blkHeight) + if err := database.PutID(s.blockIDDB, heightKey, blkID); err != nil { + return fmt.Errorf("failed to add blockID: %w", err) + } + } + + return nil +} From 643932ed63f58f68af5fdd564124808459b7d652 Mon Sep 17 00:00:00 2001 From: Dhruba Basu <7675102+dhrubabasu@users.noreply.github.com> Date: Fri, 28 Apr 2023 23:20:45 -0400 Subject: [PATCH 02/11] Update vms/platformvm/state/state.go --- vms/platformvm/state/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vms/platformvm/state/state.go b/vms/platformvm/state/state.go index 48b84c90b96..827ecaa4124 100644 --- a/vms/platformvm/state/state.go +++ b/vms/platformvm/state/state.go @@ -240,7 +240,7 @@ type state struct { blockIDDB database.Database addedBlocks map[ids.ID]stateBlk // map of blockID -> Block - blockCache cache.Cacher[ids.ID, *stateBlk] // cache of blockID -> Block. If the entry is nil, it is not in the database. + blockCache cache.Cacher[ids.ID, *stateBlk] // cache of blockID -> Block. If the entry is nil, it is not in the database blockDB database.Database validatorsDB database.Database From c6ab13a40e3f33dbce9d23ecc4ddb5e14c851bcf Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Sun, 30 Apr 2023 15:36:42 -0400 Subject: [PATCH 03/11] Add P-chain `GetBlockByHeight` API method --- vms/platformvm/client.go | 21 +++- vms/platformvm/service.go | 37 +++++++ vms/platformvm/service_test.go | 188 +++++++++++++++++++++++++++++++++ 3 files changed, 243 insertions(+), 3 deletions(-) diff --git a/vms/platformvm/client.go b/vms/platformvm/client.go index f3a1f63b1ea..7dfcc2da0a7 100644 --- a/vms/platformvm/client.go +++ b/vms/platformvm/client.go @@ -255,6 +255,8 @@ type Client interface { GetValidatorsAt(ctx context.Context, subnetID ids.ID, height uint64, options ...rpc.Option) (map[ids.NodeID]uint64, error) // GetBlock returns the block with the given id. GetBlock(ctx context.Context, blockID ids.ID, options ...rpc.Option) ([]byte, error) + // GetBlockByHeight returns the block at the given [height]. + GetBlockByHeight(ctx context.Context, height uint64, options ...rpc.Option) ([]byte, error) } // Client implementation for interacting with the P Chain endpoint @@ -857,13 +859,26 @@ func (c *client) GetValidatorsAt(ctx context.Context, subnetID ids.ID, height ui } func (c *client) GetBlock(ctx context.Context, blockID ids.ID, options ...rpc.Option) ([]byte, error) { - response := &api.FormattedBlock{} + res := &api.FormattedBlock{} if err := c.requester.SendRequest(ctx, "platform.getBlock", &api.GetBlockArgs{ BlockID: blockID, Encoding: formatting.Hex, - }, response, options...); err != nil { + }, res, options...); err != nil { return nil, err } - return formatting.Decode(response.Encoding, response.Block) + return formatting.Decode(res.Encoding, res.Block) +} + +func (c *client) GetBlockByHeight(ctx context.Context, height uint64, options ...rpc.Option) ([]byte, error) { + res := &api.FormattedBlock{} + err := c.requester.SendRequest(ctx, "platform.getBlockByHeight", &api.GetBlockByHeightArgs{ + Height: json.Uint64(height), + Encoding: formatting.HexNC, + }, res, options...) + if err != nil { + return nil, err + } + + return formatting.Decode(res.Encoding, res.Block) } diff --git a/vms/platformvm/service.go b/vms/platformvm/service.go index 305779a172d..76b0b5bbdf3 100644 --- a/vms/platformvm/service.go +++ b/vms/platformvm/service.go @@ -2595,6 +2595,43 @@ func (s *Service) GetBlock(_ *http.Request, args *api.GetBlockArgs, response *ap return nil } +// GetBlockByHeight returns the block at the given height. +func (s *Service) GetBlockByHeight(_ *http.Request, args *api.GetBlockByHeightArgs, response *api.GetBlockResponse) error { + s.vm.ctx.Log.Debug("API called", + zap.String("service", "platform"), + zap.String("method", "getBlockByHeight"), + zap.Uint64("height", uint64(args.Height)), + ) + + response.Encoding = args.Encoding + + blockID, err := s.vm.state.GetBlockID(uint64(args.Height)) + if err != nil { + return fmt.Errorf("couldn't get block at height %d: %w", args.Height, err) + } + block, err := s.vm.manager.GetStatelessBlock(blockID) + if err != nil { + s.vm.ctx.Log.Error("couldn't get accepted block", + zap.Stringer("blkID", blockID), + zap.Error(err), + ) + return fmt.Errorf("couldn't get block with id %s: %w", blockID, err) + } + + if args.Encoding == formatting.JSON { + block.InitCtx(s.vm.ctx) + response.Block = block + return nil + } + + response.Block, err = formatting.Encode(args.Encoding, block.Bytes()) + if err != nil { + return fmt.Errorf("couldn't encode block %s as string: %w", blockID, err) + } + + return nil +} + func (s *Service) getAPIUptime(staker *state.Staker) (*json.Float32, error) { // Only report uptimes that we have been actively tracking. if constants.PrimaryNetworkID != staker.SubnetID && !s.vm.TrackedSubnets.Contains(staker.SubnetID) { diff --git a/vms/platformvm/service_test.go b/vms/platformvm/service_test.go index d154718ab42..fc7b7b02875 100644 --- a/vms/platformvm/service_test.go +++ b/vms/platformvm/service_test.go @@ -13,6 +13,7 @@ import ( stdjson "encoding/json" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/api" @@ -23,6 +24,7 @@ import ( "github.com/ava-labs/avalanchego/database/manager" "github.com/ava-labs/avalanchego/database/prefixdb" "github.com/ava-labs/avalanchego/ids" + "github.com/ava-labs/avalanchego/snow" "github.com/ava-labs/avalanchego/snow/consensus/snowman" "github.com/ava-labs/avalanchego/utils/constants" "github.com/ava-labs/avalanchego/utils/crypto/secp256k1" @@ -803,3 +805,189 @@ func TestGetBlock(t *testing.T) { }) } } + +func TestServiceGetBlockByHeight(t *testing.T) { + require := require.New(t) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + blockID := ids.GenerateTestID() + blockHeight := uint64(1337) + + type test struct { + name string + serviceAndExpectedBlockFunc func(ctrl *gomock.Controller) (*Service, interface{}) + encoding formatting.Encoding + expectedErr error + } + + tests := []test{ + { + name: "block height not found", + serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + state := state.NewMockState(ctrl) + state.EXPECT().GetBlockID(blockHeight).Return(ids.Empty, database.ErrNotFound) + + manager := blockexecutor.NewMockManager(ctrl) + return &Service{ + vm: &VM{ + state: state, + manager: manager, + ctx: &snow.Context{ + Log: logging.NoLog{}, + }, + }, + }, nil + }, + encoding: formatting.Hex, + expectedErr: database.ErrNotFound, + }, + { + name: "block not found", + serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + state := state.NewMockState(ctrl) + state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) + + manager := blockexecutor.NewMockManager(ctrl) + manager.EXPECT().GetStatelessBlock(blockID).Return(nil, database.ErrNotFound) + return &Service{ + vm: &VM{ + state: state, + manager: manager, + ctx: &snow.Context{ + Log: logging.NoLog{}, + }, + }, + }, nil + }, + encoding: formatting.Hex, + expectedErr: database.ErrNotFound, + }, + { + name: "JSON format", + serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + block := blocks.NewMockBlock(ctrl) + block.EXPECT().InitCtx(gomock.Any()) + + state := state.NewMockState(ctrl) + state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) + + manager := blockexecutor.NewMockManager(ctrl) + manager.EXPECT().GetStatelessBlock(blockID).Return(block, nil) + return &Service{ + vm: &VM{ + state: state, + manager: manager, + ctx: &snow.Context{ + Log: logging.NoLog{}, + }, + }, + }, block + }, + encoding: formatting.JSON, + expectedErr: nil, + }, + { + name: "hex format", + serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + block := blocks.NewMockBlock(ctrl) + blockBytes := []byte("hi mom") + block.EXPECT().Bytes().Return(blockBytes) + + state := state.NewMockState(ctrl) + state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) + + expected, err := formatting.Encode(formatting.Hex, blockBytes) + require.NoError(err) + + manager := blockexecutor.NewMockManager(ctrl) + manager.EXPECT().GetStatelessBlock(blockID).Return(block, nil) + return &Service{ + vm: &VM{ + state: state, + manager: manager, + ctx: &snow.Context{ + Log: logging.NoLog{}, + }, + }, + }, expected + }, + encoding: formatting.Hex, + expectedErr: nil, + }, + { + name: "hexc format", + serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + block := blocks.NewMockBlock(ctrl) + blockBytes := []byte("hi mom") + block.EXPECT().Bytes().Return(blockBytes) + + state := state.NewMockState(ctrl) + state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) + + expected, err := formatting.Encode(formatting.HexC, blockBytes) + require.NoError(err) + + manager := blockexecutor.NewMockManager(ctrl) + manager.EXPECT().GetStatelessBlock(blockID).Return(block, nil) + return &Service{ + vm: &VM{ + state: state, + manager: manager, + ctx: &snow.Context{ + Log: logging.NoLog{}, + }, + }, + }, expected + }, + encoding: formatting.HexC, + expectedErr: nil, + }, + { + name: "hexnc format", + serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + block := blocks.NewMockBlock(ctrl) + blockBytes := []byte("hi mom") + block.EXPECT().Bytes().Return(blockBytes) + + state := state.NewMockState(ctrl) + state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) + + expected, err := formatting.Encode(formatting.HexNC, blockBytes) + require.NoError(err) + + manager := blockexecutor.NewMockManager(ctrl) + manager.EXPECT().GetStatelessBlock(blockID).Return(block, nil) + return &Service{ + vm: &VM{ + state: state, + manager: manager, + ctx: &snow.Context{ + Log: logging.NoLog{}, + }, + }, + }, expected + }, + encoding: formatting.HexNC, + expectedErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + service, expected := tt.serviceAndExpectedBlockFunc(ctrl) + + args := &api.GetBlockByHeightArgs{ + Height: json.Uint64(blockHeight), + Encoding: tt.encoding, + } + reply := &api.GetBlockResponse{} + err := service.GetBlockByHeight(nil, args, reply) + require.ErrorIs(err, tt.expectedErr) + if err == nil { + require.Equal(tt.encoding, reply.Encoding) + require.Equal(expected, reply.Block) + } + }) + } +} From c9cc1a63e795efb553cf79aa25f88064f5fe0b77 Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Sun, 30 Apr 2023 23:02:38 -0400 Subject: [PATCH 04/11] reduce diff --- vms/platformvm/client.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/vms/platformvm/client.go b/vms/platformvm/client.go index 7dfcc2da0a7..8ae6835f1d1 100644 --- a/vms/platformvm/client.go +++ b/vms/platformvm/client.go @@ -859,15 +859,15 @@ func (c *client) GetValidatorsAt(ctx context.Context, subnetID ids.ID, height ui } func (c *client) GetBlock(ctx context.Context, blockID ids.ID, options ...rpc.Option) ([]byte, error) { - res := &api.FormattedBlock{} + response := &api.FormattedBlock{} if err := c.requester.SendRequest(ctx, "platform.getBlock", &api.GetBlockArgs{ BlockID: blockID, Encoding: formatting.Hex, - }, res, options...); err != nil { + }, response, options...); err != nil { return nil, err } - return formatting.Decode(res.Encoding, res.Block) + return formatting.Decode(response.Encoding, response.Block) } func (c *client) GetBlockByHeight(ctx context.Context, height uint64, options ...rpc.Option) ([]byte, error) { @@ -879,6 +879,5 @@ func (c *client) GetBlockByHeight(ctx context.Context, height uint64, options .. if err != nil { return nil, err } - return formatting.Decode(res.Encoding, res.Block) } From 1cc8adfc118e37e15dc13e5b471b5f860db5e127 Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Thu, 11 May 2023 06:48:04 -0400 Subject: [PATCH 05/11] explicitly skip the lastAcceptedBlk --- vms/platformvm/state/state.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/vms/platformvm/state/state.go b/vms/platformvm/state/state.go index 827ecaa4124..a984f7a3490 100644 --- a/vms/platformvm/state/state.go +++ b/vms/platformvm/state/state.go @@ -2013,13 +2013,14 @@ func (s *state) writeMetadata() error { // TODO: Remove after next hard-fork when everybody has run this. func (s *state) populateBlockHeightIndex() error { - blk, _, err := s.GetStatelessBlock(s.lastAccepted) + lastAcceptedBlk, _, err := s.GetStatelessBlock(s.lastAccepted) if err != nil { return fmt.Errorf("failed to retrieve last accepted block from local disk: %w", err) } - heightKey := database.PackUInt64(blk.Height()) - if _, err := database.GetID(s.blockIDDB, heightKey); err == nil { + lastAcceptedHeight := lastAcceptedBlk.Height() + lastAcceptedHeightKey := database.PackUInt64(lastAcceptedHeight) + if _, err := database.GetID(s.blockIDDB, lastAcceptedHeightKey); err == nil { // If the last accepted block is in [s.blockIDDB], no backfilling is required. return nil } @@ -2043,6 +2044,12 @@ func (s *state) populateBlockHeightIndex() error { blkHeight := blkState.Blk.Height() blkID := blkState.Blk.ID() + // During the indexing process, we skip [lastAcceptedHeight] as we use it to + // determine if the index is complete. + if blkHeight == lastAcceptedHeight { + continue + } + // Populate the in-memory cache with the [blockIDCacheSize] most recent blocks if math.AbsDiff(s.currentHeight, blkHeight) <= blockIDCacheSize { s.addedBlockIDs[blkHeight] = blkID @@ -2055,5 +2062,9 @@ func (s *state) populateBlockHeightIndex() error { } } + if err := database.PutID(s.blockIDDB, lastAcceptedHeightKey, s.lastAccepted); err != nil { + return fmt.Errorf("failed to add blockID: %w", err) + } + return nil } From 4c2c194ce62d0bbc586a9196bd0d1ca09de094db Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Thu, 11 May 2023 06:51:16 -0400 Subject: [PATCH 06/11] remove cache --- vms/platformvm/state/state.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/vms/platformvm/state/state.go b/vms/platformvm/state/state.go index a984f7a3490..7c6c5df2e46 100644 --- a/vms/platformvm/state/state.go +++ b/vms/platformvm/state/state.go @@ -2050,12 +2050,6 @@ func (s *state) populateBlockHeightIndex() error { continue } - // Populate the in-memory cache with the [blockIDCacheSize] most recent blocks - if math.AbsDiff(s.currentHeight, blkHeight) <= blockIDCacheSize { - s.addedBlockIDs[blkHeight] = blkID - s.blockIDCache.Put(blkHeight, blkID) - } - heightKey := database.PackUInt64(blkHeight) if err := database.PutID(s.blockIDDB, heightKey, blkID); err != nil { return fmt.Errorf("failed to add blockID: %w", err) From 9a35388cfce8904b914584956ed0fa34cbae5501 Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Thu, 11 May 2023 06:51:43 -0400 Subject: [PATCH 07/11] nit --- vms/platformvm/state/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vms/platformvm/state/state.go b/vms/platformvm/state/state.go index 7c6c5df2e46..f5b5d05a40a 100644 --- a/vms/platformvm/state/state.go +++ b/vms/platformvm/state/state.go @@ -2015,7 +2015,7 @@ func (s *state) writeMetadata() error { func (s *state) populateBlockHeightIndex() error { lastAcceptedBlk, _, err := s.GetStatelessBlock(s.lastAccepted) if err != nil { - return fmt.Errorf("failed to retrieve last accepted block from local disk: %w", err) + return fmt.Errorf("failed to get last accepted block from local disk: %w", err) } lastAcceptedHeight := lastAcceptedBlk.Height() From 4606630c089694cb560ef14327b94ecd15acfebd Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Thu, 11 May 2023 06:57:08 -0400 Subject: [PATCH 08/11] add log --- vms/platformvm/state/state.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vms/platformvm/state/state.go b/vms/platformvm/state/state.go index f5b5d05a40a..5faac6d20ca 100644 --- a/vms/platformvm/state/state.go +++ b/vms/platformvm/state/state.go @@ -9,6 +9,7 @@ import ( "time" "github.com/google/btree" + "go.uber.org/zap" "github.com/prometheus/client_golang/prometheus" @@ -2025,6 +2026,9 @@ func (s *state) populateBlockHeightIndex() error { return nil } + s.ctx.Log.Info("populating platformvm block height index") + startTime := time.Now() + blockIterator := s.blockDB.NewIterator() defer blockIterator.Release() for blockIterator.Next() { @@ -2060,5 +2064,7 @@ func (s *state) populateBlockHeightIndex() error { return fmt.Errorf("failed to add blockID: %w", err) } + s.ctx.Log.Info("populated platformvm block height index", zap.Duration("elapsed", time.Since(startTime))) + return nil } From d63329cef534d067ce0d66ee38d9e91d805e627f Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Thu, 11 May 2023 11:36:07 -0400 Subject: [PATCH 09/11] import nit --- vms/platformvm/state/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/vms/platformvm/state/state.go b/vms/platformvm/state/state.go index 5faac6d20ca..504e21e3d8c 100644 --- a/vms/platformvm/state/state.go +++ b/vms/platformvm/state/state.go @@ -9,6 +9,7 @@ import ( "time" "github.com/google/btree" + "go.uber.org/zap" "github.com/prometheus/client_golang/prometheus" From 34b90114996bed911600aa52e4e05cc1056610e2 Mon Sep 17 00:00:00 2001 From: dhrubabasu <7675102+dhrubabasu@users.noreply.github.com> Date: Mon, 15 May 2023 06:39:58 -0400 Subject: [PATCH 10/11] nit --- vms/platformvm/service_test.go | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/vms/platformvm/service_test.go b/vms/platformvm/service_test.go index fc7b7b02875..677899f7daa 100644 --- a/vms/platformvm/service_test.go +++ b/vms/platformvm/service_test.go @@ -807,7 +807,6 @@ func TestGetBlock(t *testing.T) { } func TestServiceGetBlockByHeight(t *testing.T) { - require := require.New(t) ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -816,7 +815,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { type test struct { name string - serviceAndExpectedBlockFunc func(ctrl *gomock.Controller) (*Service, interface{}) + serviceAndExpectedBlockFunc func(t *testing.T, ctrl *gomock.Controller) (*Service, interface{}) encoding formatting.Encoding expectedErr error } @@ -824,7 +823,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { tests := []test{ { name: "block height not found", - serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + serviceAndExpectedBlockFunc: func(_ *testing.T, ctrl *gomock.Controller) (*Service, interface{}) { state := state.NewMockState(ctrl) state.EXPECT().GetBlockID(blockHeight).Return(ids.Empty, database.ErrNotFound) @@ -844,7 +843,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { }, { name: "block not found", - serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + serviceAndExpectedBlockFunc: func(_ *testing.T, ctrl *gomock.Controller) (*Service, interface{}) { state := state.NewMockState(ctrl) state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) @@ -865,7 +864,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { }, { name: "JSON format", - serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + serviceAndExpectedBlockFunc: func(_ *testing.T, ctrl *gomock.Controller) (*Service, interface{}) { block := blocks.NewMockBlock(ctrl) block.EXPECT().InitCtx(gomock.Any()) @@ -889,7 +888,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { }, { name: "hex format", - serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + serviceAndExpectedBlockFunc: func(t *testing.T, ctrl *gomock.Controller) (*Service, interface{}) { block := blocks.NewMockBlock(ctrl) blockBytes := []byte("hi mom") block.EXPECT().Bytes().Return(blockBytes) @@ -898,7 +897,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) expected, err := formatting.Encode(formatting.Hex, blockBytes) - require.NoError(err) + require.NoError(t, err) manager := blockexecutor.NewMockManager(ctrl) manager.EXPECT().GetStatelessBlock(blockID).Return(block, nil) @@ -917,7 +916,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { }, { name: "hexc format", - serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + serviceAndExpectedBlockFunc: func(t *testing.T, ctrl *gomock.Controller) (*Service, interface{}) { block := blocks.NewMockBlock(ctrl) blockBytes := []byte("hi mom") block.EXPECT().Bytes().Return(blockBytes) @@ -926,7 +925,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) expected, err := formatting.Encode(formatting.HexC, blockBytes) - require.NoError(err) + require.NoError(t, err) manager := blockexecutor.NewMockManager(ctrl) manager.EXPECT().GetStatelessBlock(blockID).Return(block, nil) @@ -945,7 +944,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { }, { name: "hexnc format", - serviceAndExpectedBlockFunc: func(ctrl *gomock.Controller) (*Service, interface{}) { + serviceAndExpectedBlockFunc: func(t *testing.T, ctrl *gomock.Controller) (*Service, interface{}) { block := blocks.NewMockBlock(ctrl) blockBytes := []byte("hi mom") block.EXPECT().Bytes().Return(blockBytes) @@ -954,7 +953,7 @@ func TestServiceGetBlockByHeight(t *testing.T) { state.EXPECT().GetBlockID(blockHeight).Return(blockID, nil) expected, err := formatting.Encode(formatting.HexNC, blockBytes) - require.NoError(err) + require.NoError(t, err) manager := blockexecutor.NewMockManager(ctrl) manager.EXPECT().GetStatelessBlock(blockID).Return(block, nil) @@ -975,7 +974,9 @@ func TestServiceGetBlockByHeight(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - service, expected := tt.serviceAndExpectedBlockFunc(ctrl) + require := require.New(t) + + service, expected := tt.serviceAndExpectedBlockFunc(t, ctrl) args := &api.GetBlockByHeightArgs{ Height: json.Uint64(blockHeight), @@ -984,10 +985,11 @@ func TestServiceGetBlockByHeight(t *testing.T) { reply := &api.GetBlockResponse{} err := service.GetBlockByHeight(nil, args, reply) require.ErrorIs(err, tt.expectedErr) - if err == nil { - require.Equal(tt.encoding, reply.Encoding) - require.Equal(expected, reply.Block) + if tt.expectedErr == nil { + return } + require.Equal(tt.encoding, reply.Encoding) + require.Equal(expected, reply.Block) }) } } From 6868edf3ca0facb6c09304b3124dbbc35b5ecc66 Mon Sep 17 00:00:00 2001 From: Stephen Buttolph Date: Wed, 31 May 2023 00:22:28 -0400 Subject: [PATCH 11/11] nit --- vms/platformvm/service_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/vms/platformvm/service_test.go b/vms/platformvm/service_test.go index f2cf4f77539..f671726a174 100644 --- a/vms/platformvm/service_test.go +++ b/vms/platformvm/service_test.go @@ -14,6 +14,7 @@ import ( stdjson "encoding/json" "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" "github.com/ava-labs/avalanchego/api"