Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

P-Chain merkledb -- include txs in merkleized state #2398

Merged
merged 25 commits into from
Dec 5, 2023
Merged
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
20f7007
add newView to merkleState; refactor logMerkleRoot to not return error
Nov 20, 2023
91d9cda
add NewView to State interface
Nov 20, 2023
8785c55
wip add method GetMerkleChanges to diff
Nov 21, 2023
28384fb
WIP add replacement for writeCurrentStakers
Nov 22, 2023
0a92fc2
refactor writeCurrentStakers block
Nov 27, 2023
a3c8258
WIP add replacement for writePendingStakers
Nov 27, 2023
be4aec8
refactor writePendingStakers block
Nov 27, 2023
2760af7
Merge branch 'pchain_merkleDB' into pchain_merkleDB_view
Nov 27, 2023
a7b758b
implement NewView on diff
Nov 27, 2023
afa056b
Merge branch 'pchain_merkleDB' into pchain_merkleDB_view
Nov 28, 2023
192072c
update NewView interface
Nov 28, 2023
256b278
comment
Nov 28, 2023
6f832cc
unexport function
Nov 29, 2023
6fc395e
Merge branch 'pchain_merkleDB' into pchain_merkleDB_view
Nov 29, 2023
98dfdf1
Merge branch 'pchain_merkleDB' into pchain_merkleDB_view
Nov 29, 2023
cadce70
update writeMetadata portion of getMerkleChanges to include last acce…
Nov 29, 2023
d98b52b
include txs in merkleized state
Nov 29, 2023
0ae9ebb
Merge branch 'pchain_merkleDB' into pchain_merkleDB_view
Nov 30, 2023
673985b
Merge branch 'pchain_merkleDB_view' into pchain_merkleDB_merkleize_txs
Nov 30, 2023
1c501d6
appease linter
Nov 30, 2023
7d812e6
Merge branch 'pchain_merkleDB' into pchain_merkleDB_merkleize_txs
Dec 1, 2023
1d6f682
Merge branch 'pchain_merkleDB' into pchain_merkleDB_merkleize_txs
Dec 5, 2023
7cfe789
Merge branch 'pchain_merkleDB' into pchain_merkleDB_merkleize_txs
Dec 5, 2023
3442bca
revert unneeded change
Dec 5, 2023
db27c17
remove txDB
Dec 5, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'pchain_merkleDB' into pchain_merkleDB_view
Dan Laine committed Nov 28, 2023

Verified

This commit was signed with the committer’s verified signature.
kenjis kenjis
commit afa056b54825f44d8a77265ea2853b9f2443364b
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@ The minimum recommended hardware specification for nodes connected to Mainnet is
- CPU: Equivalent of 8 AWS vCPU
- RAM: 16 GiB
- Storage: 1 TiB
- Nodes running for very long periods of time or nodes with custom configurations may observe higher storage requirements.
- OS: Ubuntu 20.04/22.04 or macOS >= 12
- Network: Reliable IPv4 or IPv6 network connection, with an open public port.

57 changes: 24 additions & 33 deletions chains/manager.go
Original file line number Diff line number Diff line change
@@ -878,21 +878,17 @@ func (m *manager) createAvalancheChain(

// create bootstrap gear
bootstrapCfg := smbootstrap.Config{
Config: common.Config{
Ctx: ctx,
Beacons: vdrs,
SampleK: sampleK,
Alpha: bootstrapWeight/2 + 1, // must be > 50%
StartupTracker: startupTracker,
Sender: snowmanMessageSender,
BootstrapTracker: sb,
Timer: h,
AncestorsMaxContainersReceived: m.BootstrapAncestorsMaxContainersReceived,
SharedCfg: &common.SharedConfig{},
},
AllGetsServer: snowGetHandler,
Blocked: blockBlocker,
VM: vmWrappingProposerVM,
AllGetsServer: snowGetHandler,
Ctx: ctx,
Beacons: vdrs,
SampleK: sampleK,
StartupTracker: startupTracker,
Sender: snowmanMessageSender,
BootstrapTracker: sb,
Timer: h,
AncestorsMaxContainersReceived: m.BootstrapAncestorsMaxContainersReceived,
Blocked: blockBlocker,
VM: vmWrappingProposerVM,
}
var snowmanBootstrapper common.BootstrapableEngine
snowmanBootstrapper, err = smbootstrap.New(
@@ -1224,24 +1220,19 @@ func (m *manager) createSnowmanChain(
}

// create bootstrap gear
alpha := bootstrapWeight/2 + 1 // must be > 50%
bootstrapCfg := smbootstrap.Config{
Config: common.Config{
Ctx: ctx,
Beacons: beacons,
SampleK: sampleK,
StartupTracker: startupTracker,
Alpha: alpha,
Sender: messageSender,
BootstrapTracker: sb,
Timer: h,
AncestorsMaxContainersReceived: m.BootstrapAncestorsMaxContainersReceived,
SharedCfg: &common.SharedConfig{},
},
AllGetsServer: snowGetHandler,
Blocked: blocked,
VM: vm,
Bootstrapped: bootstrapFunc,
AllGetsServer: snowGetHandler,
Ctx: ctx,
Beacons: beacons,
SampleK: sampleK,
StartupTracker: startupTracker,
Sender: messageSender,
BootstrapTracker: sb,
Timer: h,
AncestorsMaxContainersReceived: m.BootstrapAncestorsMaxContainersReceived,
Blocked: blocked,
VM: vm,
Bootstrapped: bootstrapFunc,
}
var bootstrapper common.BootstrapableEngine
bootstrapper, err = smbootstrap.New(
@@ -1264,7 +1255,7 @@ func (m *manager) createSnowmanChain(
messageSender,
beacons,
sampleK,
alpha,
bootstrapWeight/2+1, // must be > 50%
m.StateSyncBeacons,
vm,
)
26 changes: 12 additions & 14 deletions codec/hierarchycodec/codec.go
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ import (

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/reflectcodec"
"github.com/ava-labs/avalanchego/utils/bimap"
"github.com/ava-labs/avalanchego/utils/wrappers"
)

@@ -42,20 +43,18 @@ type typeID struct {
type hierarchyCodec struct {
codec.Codec

lock sync.RWMutex
currentGroupID uint16
nextTypeID uint16
typeIDToType map[typeID]reflect.Type
typeToTypeID map[reflect.Type]typeID
lock sync.RWMutex
currentGroupID uint16
nextTypeID uint16
registeredTypes *bimap.BiMap[typeID, reflect.Type]
}

// New returns a new, concurrency-safe codec
func New(tagNames []string, maxSliceLen uint32) Codec {
hCodec := &hierarchyCodec{
currentGroupID: 0,
nextTypeID: 0,
typeIDToType: map[typeID]reflect.Type{},
typeToTypeID: map[reflect.Type]typeID{},
currentGroupID: 0,
nextTypeID: 0,
registeredTypes: bimap.New[typeID, reflect.Type](),
}
hCodec.Codec = reflectcodec.New(hCodec, tagNames, maxSliceLen)
return hCodec
@@ -88,7 +87,7 @@ func (c *hierarchyCodec) RegisterType(val interface{}) error {
defer c.lock.Unlock()

valType := reflect.TypeOf(val)
if _, exists := c.typeToTypeID[valType]; exists {
if c.registeredTypes.HasValue(valType) {
return fmt.Errorf("%w: %v", codec.ErrDuplicateType, valType)
}

@@ -98,8 +97,7 @@ func (c *hierarchyCodec) RegisterType(val interface{}) error {
}
c.nextTypeID++

c.typeIDToType[valTypeID] = valType
c.typeToTypeID[valType] = valTypeID
c.registeredTypes.Put(valTypeID, valType)
return nil
}

@@ -112,7 +110,7 @@ func (c *hierarchyCodec) PackPrefix(p *wrappers.Packer, valueType reflect.Type)
c.lock.RLock()
defer c.lock.RUnlock()

typeID, ok := c.typeToTypeID[valueType] // Get the type ID of the value being marshaled
typeID, ok := c.registeredTypes.GetKey(valueType) // Get the type ID of the value being marshaled
if !ok {
return fmt.Errorf("can't marshal unregistered type %q", valueType)
}
@@ -136,7 +134,7 @@ func (c *hierarchyCodec) UnpackPrefix(p *wrappers.Packer, valueType reflect.Type
typeID: typeIDShort,
}
// Get a type that implements the interface
implementingType, ok := c.typeIDToType[t]
implementingType, ok := c.registeredTypes.GetValue(t)
if !ok {
return reflect.Value{}, fmt.Errorf("couldn't unmarshal interface: unknown type ID %+v", t)
}
22 changes: 10 additions & 12 deletions codec/linearcodec/codec.go
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ import (

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/reflectcodec"
"github.com/ava-labs/avalanchego/utils/bimap"
"github.com/ava-labs/avalanchego/utils/wrappers"
)

@@ -36,19 +37,17 @@ type Codec interface {
type linearCodec struct {
codec.Codec

lock sync.RWMutex
nextTypeID uint32
typeIDToType map[uint32]reflect.Type
typeToTypeID map[reflect.Type]uint32
lock sync.RWMutex
nextTypeID uint32
registeredTypes *bimap.BiMap[uint32, reflect.Type]
}

// New returns a new, concurrency-safe codec; it allow to specify
// both tagNames and maxSlicelenght
func New(tagNames []string, maxSliceLen uint32) Codec {
hCodec := &linearCodec{
nextTypeID: 0,
typeIDToType: map[uint32]reflect.Type{},
typeToTypeID: map[reflect.Type]uint32{},
nextTypeID: 0,
registeredTypes: bimap.New[uint32, reflect.Type](),
}
hCodec.Codec = reflectcodec.New(hCodec, tagNames, maxSliceLen)
return hCodec
@@ -78,12 +77,11 @@ func (c *linearCodec) RegisterType(val interface{}) error {
defer c.lock.Unlock()

valType := reflect.TypeOf(val)
if _, exists := c.typeToTypeID[valType]; exists {
if c.registeredTypes.HasValue(valType) {
return fmt.Errorf("%w: %v", codec.ErrDuplicateType, valType)
}

c.typeIDToType[c.nextTypeID] = valType
c.typeToTypeID[valType] = c.nextTypeID
c.registeredTypes.Put(c.nextTypeID, valType)
c.nextTypeID++
return nil
}
@@ -97,7 +95,7 @@ func (c *linearCodec) PackPrefix(p *wrappers.Packer, valueType reflect.Type) err
c.lock.RLock()
defer c.lock.RUnlock()

typeID, ok := c.typeToTypeID[valueType] // Get the type ID of the value being marshaled
typeID, ok := c.registeredTypes.GetKey(valueType) // Get the type ID of the value being marshaled
if !ok {
return fmt.Errorf("can't marshal unregistered type %q", valueType)
}
@@ -114,7 +112,7 @@ func (c *linearCodec) UnpackPrefix(p *wrappers.Packer, valueType reflect.Type) (
return reflect.Value{}, fmt.Errorf("couldn't unmarshal interface: %w", p.Err)
}
// Get a type that implements the interface
implementingType, ok := c.typeIDToType[typeID]
implementingType, ok := c.registeredTypes.GetValue(typeID)
if !ok {
return reflect.Value{}, fmt.Errorf("couldn't unmarshal interface: unknown type ID %d", typeID)
}
9 changes: 6 additions & 3 deletions genesis/config.go
Original file line number Diff line number Diff line change
@@ -17,6 +17,7 @@ import (
"github.com/ava-labs/avalanchego/utils/constants"
"github.com/ava-labs/avalanchego/utils/formatting/address"
"github.com/ava-labs/avalanchego/utils/math"
"github.com/ava-labs/avalanchego/vms/platformvm/signer"
)

var (
@@ -58,9 +59,10 @@ func (a Allocation) Less(other Allocation) bool {
}

type Staker struct {
NodeID ids.NodeID `json:"nodeID"`
RewardAddress ids.ShortID `json:"rewardAddress"`
DelegationFee uint32 `json:"delegationFee"`
NodeID ids.NodeID `json:"nodeID"`
RewardAddress ids.ShortID `json:"rewardAddress"`
DelegationFee uint32 `json:"delegationFee"`
Signer *signer.ProofOfPossession `json:"signer,omitempty"`
}

func (s Staker) Unparse(networkID uint32) (UnparsedStaker, error) {
@@ -73,6 +75,7 @@ func (s Staker) Unparse(networkID uint32) (UnparsedStaker, error) {
NodeID: s.NodeID,
RewardAddress: avaxAddr,
DelegationFee: s.DelegationFee,
Signer: s.Signer,
}, err
}

1 change: 1 addition & 0 deletions genesis/genesis.go
Original file line number Diff line number Diff line change
@@ -430,6 +430,7 @@ func FromConfig(config *Config) ([]byte, ids.ID, error) {
},
Staked: utxos,
ExactDelegationFee: &delegationFee,
Signer: staker.Signer,
},
)
}
9 changes: 6 additions & 3 deletions genesis/unparsed_config.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import (

"github.com/ava-labs/avalanchego/ids"
"github.com/ava-labs/avalanchego/utils/formatting/address"
"github.com/ava-labs/avalanchego/vms/platformvm/signer"
)

var errInvalidETHAddress = errors.New("invalid eth address")
@@ -54,15 +55,17 @@ func (ua UnparsedAllocation) Parse() (Allocation, error) {
}

type UnparsedStaker struct {
NodeID ids.NodeID `json:"nodeID"`
RewardAddress string `json:"rewardAddress"`
DelegationFee uint32 `json:"delegationFee"`
NodeID ids.NodeID `json:"nodeID"`
RewardAddress string `json:"rewardAddress"`
DelegationFee uint32 `json:"delegationFee"`
Signer *signer.ProofOfPossession `json:"signer,omitempty"`
}

func (us UnparsedStaker) Parse() (Staker, error) {
s := Staker{
NodeID: us.NodeID,
DelegationFee: us.DelegationFee,
Signer: us.Signer,
}

_, _, avaxAddrBytes, err := address.Parse(us.RewardAddress)
25 changes: 13 additions & 12 deletions x/sync/peer_tracker.go → network/p2p/peer_tracker.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// Copyright (C) 2019-2023, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package sync
package p2p

import (
"math"
@@ -45,7 +45,7 @@ type peerInfo struct {
// Tracks the bandwidth of responses coming from peers,
// preferring to contact peers with known good bandwidth, connecting
// to new peers with an exponentially decaying probability.
type peerTracker struct {
type PeerTracker struct {
// Lock to protect concurrent access to the peer tracker
lock sync.Mutex
// All peers we are connected to
@@ -64,12 +64,12 @@ type peerTracker struct {
averageBandwidthMetric prometheus.Gauge
}

func newPeerTracker(
func NewPeerTracker(
log logging.Logger,
metricsNamespace string,
registerer prometheus.Registerer,
) (*peerTracker, error) {
t := &peerTracker{
) (*PeerTracker, error) {
t := &PeerTracker{
peers: make(map[ids.NodeID]*peerInfo),
trackedPeers: make(set.Set[ids.NodeID]),
responsivePeers: make(set.Set[ids.NodeID]),
@@ -112,7 +112,7 @@ func newPeerTracker(
// Returns true if we're not connected to enough peers.
// Otherwise returns true probabilistically based on the number of tracked peers.
// Assumes p.lock is held.
func (p *peerTracker) shouldTrackNewPeer() bool {
func (p *PeerTracker) shouldTrackNewPeer() bool {
numResponsivePeers := p.responsivePeers.Len()
if numResponsivePeers < desiredMinResponsivePeers {
return true
@@ -137,11 +137,12 @@ func (p *peerTracker) shouldTrackNewPeer() bool {
return rand.Float64() < newPeerProbability // #nosec G404
}

// TODO get rid of minVersion
// Returns a peer that we're connected to.
// If we should track more peers, returns a random peer with version >= [minVersion], if any exist.
// Otherwise, with probability [randomPeerProbability] returns a random peer from [p.responsivePeers].
// With probability [1-randomPeerProbability] returns the peer in [p.bandwidthHeap] with the highest bandwidth.
func (p *peerTracker) GetAnyPeer(minVersion *version.Application) (ids.NodeID, bool) {
func (p *PeerTracker) GetAnyPeer(minVersion *version.Application) (ids.NodeID, bool) {
p.lock.Lock()
defer p.lock.Unlock()

@@ -187,7 +188,7 @@ func (p *peerTracker) GetAnyPeer(minVersion *version.Application) (ids.NodeID, b
}

// Record that we sent a request to [nodeID].
func (p *peerTracker) TrackPeer(nodeID ids.NodeID) {
func (p *PeerTracker) TrackPeer(nodeID ids.NodeID) {
p.lock.Lock()
defer p.lock.Unlock()

@@ -197,7 +198,7 @@ func (p *peerTracker) TrackPeer(nodeID ids.NodeID) {

// Record that we observed that [nodeID]'s bandwidth is [bandwidth].
// Adds the peer's bandwidth averager to the bandwidth heap.
func (p *peerTracker) TrackBandwidth(nodeID ids.NodeID, bandwidth float64) {
func (p *PeerTracker) TrackBandwidth(nodeID ids.NodeID, bandwidth float64) {
p.lock.Lock()
defer p.lock.Unlock()

@@ -229,7 +230,7 @@ func (p *peerTracker) TrackBandwidth(nodeID ids.NodeID, bandwidth float64) {
}

// Connected should be called when [nodeID] connects to this node
func (p *peerTracker) Connected(nodeID ids.NodeID, nodeVersion *version.Application) {
func (p *PeerTracker) Connected(nodeID ids.NodeID, nodeVersion *version.Application) {
p.lock.Lock()
defer p.lock.Unlock()

@@ -264,7 +265,7 @@ func (p *peerTracker) Connected(nodeID ids.NodeID, nodeVersion *version.Applicat
}

// Disconnected should be called when [nodeID] disconnects from this node
func (p *peerTracker) Disconnected(nodeID ids.NodeID) {
func (p *PeerTracker) Disconnected(nodeID ids.NodeID) {
p.lock.Lock()
defer p.lock.Unlock()

@@ -277,7 +278,7 @@ func (p *peerTracker) Disconnected(nodeID ids.NodeID) {
}

// Returns the number of peers the node is connected to.
func (p *peerTracker) Size() int {
func (p *PeerTracker) Size() int {
p.lock.Lock()
defer p.lock.Unlock()

Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.