diff --git a/beacon_chain/beacon_chain_db.nim b/beacon_chain/beacon_chain_db.nim index 57fc83ae60..407b3ae737 100644 --- a/beacon_chain/beacon_chain_db.nim +++ b/beacon_chain/beacon_chain_db.nim @@ -76,7 +76,7 @@ type ## database - this may have a number of "natural" causes such as switching ## between different versions of the client and accidentally using an old ## database. - db: SqStoreRef + db*: SqStoreRef v0: BeaconChainDBV0 genesisDeposits*: DepositsSeq @@ -269,25 +269,27 @@ proc loadImmutableValidators(vals: DbSeq[ImmutableValidatorDataDb2]): seq[Immuta proc new*(T: type BeaconChainDB, dir: string, inMemory = false, + readOnly = false ): BeaconChainDB = var db = if inMemory: - SqStoreRef.init("", "test", inMemory = true).expect( + SqStoreRef.init("", "test", readOnly = readOnly, inMemory = true).expect( "working database (out of memory?)") else: let s = secureCreatePath(dir) doAssert s.isOk # TODO(zah) Handle this in a better way SqStoreRef.init( - dir, "nbc", manualCheckpoint = true).expectDb() + dir, "nbc", readOnly = readOnly, manualCheckpoint = true).expectDb() - # Remove the deposits table we used before we switched - # to storing only deposit contract checkpoints - if db.exec("DROP TABLE IF EXISTS deposits;").isErr: - debug "Failed to drop the deposits table" + if not readOnly: + # Remove the deposits table we used before we switched + # to storing only deposit contract checkpoints + if db.exec("DROP TABLE IF EXISTS deposits;").isErr: + debug "Failed to drop the deposits table" - # An old pubkey->index mapping that hasn't been used on any mainnet release - if db.exec("DROP TABLE IF EXISTS validatorIndexFromPubKey;").isErr: - debug "Failed to drop the validatorIndexFromPubKey table" + # An old pubkey->index mapping that hasn't been used on any mainnet release + if db.exec("DROP TABLE IF EXISTS validatorIndexFromPubKey;").isErr: + debug "Failed to drop the validatorIndexFromPubKey table" var # V0 compatibility tables - these were created WITHOUT ROWID which is slow diff --git a/beacon_chain/beacon_chain_db_immutable.nim b/beacon_chain/beacon_chain_db_immutable.nim index 16e5fd9ee4..38701728e9 100644 --- a/beacon_chain/beacon_chain_db_immutable.nim +++ b/beacon_chain/beacon_chain_db_immutable.nim @@ -107,10 +107,8 @@ type ## Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation*: - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation*: - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation*: EpochParticipationFlags + current_epoch_participation*: EpochParticipationFlags # Finality justification_bits*: uint8 ##\ @@ -169,10 +167,8 @@ type ## Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation*: - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation*: - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation*: EpochParticipationFlags + current_epoch_participation*: EpochParticipationFlags # Finality justification_bits*: uint8 ##\ diff --git a/beacon_chain/consensus_object_pools/blockchain_dag.nim b/beacon_chain/consensus_object_pools/blockchain_dag.nim index 1ba0609baf..6393abf5a1 100644 --- a/beacon_chain/consensus_object_pools/blockchain_dag.nim +++ b/beacon_chain/consensus_object_pools/blockchain_dag.nim @@ -546,9 +546,10 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB, # Pruning metadata dag.lastPrunePoint = dag.finalizedHead - # Fill validator key cache in case we're loading an old database that doesn't - # have a cache - dag.updateValidatorKeys(getStateField(dag.headState.data, validators).asSeq()) + if not dag.db.db.readOnly: + # Fill validator key cache in case we're loading an old database that doesn't + # have a cache + dag.updateValidatorKeys(getStateField(dag.headState.data, validators).asSeq()) withState(dag.headState.data): when stateFork >= BeaconStateFork.Altair: diff --git a/beacon_chain/spec/beaconstate.nim b/beacon_chain/spec/beaconstate.nim index 1a41b6dd77..e93b45bf20 100644 --- a/beacon_chain/spec/beaconstate.nim +++ b/beacon_chain/spec/beaconstate.nim @@ -119,6 +119,39 @@ func initiate_validator_exit*(cfg: RuntimeConfig, state: var ForkyBeaconState, validator.withdrawable_epoch = validator.exit_epoch + cfg.MIN_VALIDATOR_WITHDRAWABILITY_DELAY +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slash_validator +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#modified-slash_validator +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#modified-slash_validator +proc get_slashing_penalty*(state: ForkyBeaconState, + validator_effective_balance: Gwei): Gwei = + # TODO Consider whether this is better than splitting the functions apart; in + # each case, tradeoffs. Here, it's just changing a couple of constants. + when state is phase0.BeaconState: + validator_effective_balance div MIN_SLASHING_PENALTY_QUOTIENT + elif state is altair.BeaconState: + validator_effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR + elif state is merge.BeaconState: + validator_effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_MERGE + else: + raiseAssert "invalid BeaconState type" + +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slash_validator +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#modified-slash_validator +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#modified-slash_validator +proc get_whistleblower_reward*(validator_effective_balance: Gwei): Gwei = + validator_effective_balance div WHISTLEBLOWER_REWARD_QUOTIENT + +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slash_validator +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#modified-slash_validator +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#modified-slash_validator +proc get_proposer_reward(state: ForkyBeaconState, whistleblower_reward: Gwei): Gwei = + when state is phase0.BeaconState: + whistleblower_reward div PROPOSER_REWARD_QUOTIENT + elif state is altair.BeaconState or state is merge.BeaconState: + whistleblower_reward * PROPOSER_WEIGHT div WEIGHT_DENOMINATOR + else: + raiseAssert "invalid BeaconState type" + # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slash_validator # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#modified-slash_validator # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#modified-slash_validator @@ -145,19 +178,8 @@ proc slash_validator*( state.slashings[int(epoch mod EPOCHS_PER_SLASHINGS_VECTOR)] += validator.effective_balance - # TODO Consider whether this is better than splitting the functions apart; in - # each case, tradeoffs. Here, it's just changing a couple of constants. - when state is phase0.BeaconState: - decrease_balance(state, slashed_index, - validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT) - elif state is altair.BeaconState: - decrease_balance(state, slashed_index, - validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR) - elif state is merge.BeaconState: - decrease_balance(state, slashed_index, - validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_MERGE) - else: - raiseAssert "invalid BeaconState type" + decrease_balance(state, slashed_index, + get_slashing_penalty(state, validator.effective_balance)) # The rest doesn't make sense without there being any proposer index, so skip let proposer_index = get_beacon_proposer_index(state, cache) @@ -169,15 +191,8 @@ proc slash_validator*( let # Spec has whistleblower_index as optional param, but it's never used. whistleblower_index = proposer_index.get - whistleblower_reward = - (validator.effective_balance div WHISTLEBLOWER_REWARD_QUOTIENT).Gwei - proposer_reward = - when state is phase0.BeaconState: - whistleblower_reward div PROPOSER_REWARD_QUOTIENT - elif state is altair.BeaconState or state is merge.BeaconState: - whistleblower_reward * PROPOSER_WEIGHT div WEIGHT_DENOMINATOR - else: - raiseAssert "invalid BeaconState type" + whistleblower_reward = get_whistleblower_reward(validator.effective_balance) + proposer_reward = get_proposer_reward(state, whistleblower_reward) increase_balance(state, proposer_index.get, proposer_reward) # TODO: evaluate if spec bug / underflow can be triggered @@ -623,6 +638,31 @@ proc check_attestation*( ok() +proc get_proposer_reward*(state: ForkyBeaconState, + attestation: SomeAttestation, + base_reward_per_increment: Gwei, + cache: var StateCache, + epoch_participation: var EpochParticipationFlags): uint64 = + let participation_flag_indices = get_attestation_participation_flag_indices( + state, attestation.data, state.slot - attestation.data.slot) + for index in get_attesting_indices( + state, attestation.data, attestation.aggregation_bits, cache): + for flag_index, weight in PARTICIPATION_FLAG_WEIGHTS: + if flag_index in participation_flag_indices and + not has_flag(epoch_participation.asSeq[index], flag_index): + epoch_participation.asSeq[index] = + add_flag(epoch_participation.asSeq[index], flag_index) + # these are all valid; TODO statically verify or do it type-safely + result += get_base_reward( + state, index, base_reward_per_increment) * weight.uint64 + epoch_participation.clearCache() + + let proposer_reward_denominator = + (WEIGHT_DENOMINATOR.uint64 - PROPOSER_WEIGHT.uint64) * + WEIGHT_DENOMINATOR.uint64 div PROPOSER_WEIGHT.uint64 + + return result div proposer_reward_denominator + proc process_attestation*( state: var ForkyBeaconState, attestation: SomeAttestation, flags: UpdateFlags, base_reward_per_increment: Gwei, cache: var StateCache): @@ -656,31 +696,8 @@ proc process_attestation*( # Altair and Merge template updateParticipationFlags(epoch_participation: untyped) = - var proposer_reward_numerator = 0'u64 - - # Participation flag indices - let participation_flag_indices = - get_attestation_participation_flag_indices( - state, attestation.data, state.slot - attestation.data.slot) - - for index in get_attesting_indices( - state, attestation.data, attestation.aggregation_bits, cache): - for flag_index, weight in PARTICIPATION_FLAG_WEIGHTS: - if flag_index in participation_flag_indices and - not has_flag(epoch_participation.asSeq[index], flag_index): - epoch_participation.asSeq[index] = - add_flag(epoch_participation.asSeq[index], flag_index) - - # these are all valid; TODO statically verify or do it type-safely - proposer_reward_numerator += get_base_reward( - state, index, base_reward_per_increment) * weight.uint64 - epoch_participation.clearCache() - - # Reward proposer - let - # TODO use correct type at source - proposer_reward_denominator = (WEIGHT_DENOMINATOR.uint64 - PROPOSER_WEIGHT.uint64) * WEIGHT_DENOMINATOR.uint64 div PROPOSER_WEIGHT.uint64 - proposer_reward = Gwei(proposer_reward_numerator div proposer_reward_denominator) + let proposer_reward = get_proposer_reward( + state, attestation, base_reward_per_increment, cache, epoch_participation) increase_balance(state, proposer_index.get, proposer_reward) when state is phase0.BeaconState: @@ -777,8 +794,7 @@ func translate_participation( proc upgrade_to_altair*(cfg: RuntimeConfig, pre: phase0.BeaconState): ref altair.BeaconState = var - empty_participation = - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]() + empty_participation = EpochParticipationFlags() inactivity_scores = HashList[uint64, Limit VALIDATOR_REGISTRY_LIMIT]() doAssert empty_participation.data.setLen(pre.validators.len) diff --git a/beacon_chain/spec/datatypes/altair.nim b/beacon_chain/spec/datatypes/altair.nim index 402d17c8f0..95b1d6b1a7 100644 --- a/beacon_chain/spec/datatypes/altair.nim +++ b/beacon_chain/spec/datatypes/altair.nim @@ -78,6 +78,9 @@ type # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#custom-types ParticipationFlags* = uint8 + EpochParticipationFlags* = + HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] + # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#syncaggregate SyncAggregate* = object sync_committee_bits*: BitArray[SYNC_COMMITTEE_SIZE] @@ -218,10 +221,8 @@ type ## Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation*: - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation*: - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation*: EpochParticipationFlags + current_epoch_participation*: EpochParticipationFlags # Finality justification_bits*: uint8 ##\ diff --git a/beacon_chain/spec/datatypes/merge.nim b/beacon_chain/spec/datatypes/merge.nim index 29ef9e8c00..2a9fab0407 100644 --- a/beacon_chain/spec/datatypes/merge.nim +++ b/beacon_chain/spec/datatypes/merge.nim @@ -125,10 +125,8 @@ type ## Per-epoch sums of slashed effective balances # Participation - previous_epoch_participation*: - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] - current_epoch_participation*: - HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT] + previous_epoch_participation*: EpochParticipationFlags + current_epoch_participation*: EpochParticipationFlags # Finality justification_bits*: uint8 ##\ diff --git a/beacon_chain/spec/forks.nim b/beacon_chain/spec/forks.nim index 5e50a2bd04..18a1b6ace8 100644 --- a/beacon_chain/spec/forks.nim +++ b/beacon_chain/spec/forks.nim @@ -218,9 +218,11 @@ template withState*(x: ForkedHashedBeaconState, body: untyped): untyped = template withEpochInfo*(x: ForkedEpochInfo, body: untyped): untyped = case x.kind of EpochInfoFork.Phase0: + const infoFork {.inject.} = EpochInfoFork.Phase0 template info: untyped {.inject.} = x.phase0Data body of EpochInfoFork.Altair: + const infoFork {.inject.} = EpochInfoFork.Altair template info: untyped {.inject.} = x.altairData body diff --git a/beacon_chain/spec/state_transition_block.nim b/beacon_chain/spec/state_transition_block.nim index 6324de39d0..0e6f47670f 100644 --- a/beacon_chain/spec/state_transition_block.nim +++ b/beacon_chain/spec/state_transition_block.nim @@ -124,7 +124,7 @@ func is_slashable_validator(validator: Validator, epoch: Epoch): bool = # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#proposer-slashings proc check_proposer_slashing*( - state: var ForkyBeaconState, proposer_slashing: SomeProposerSlashing, + state: ForkyBeaconState, proposer_slashing: SomeProposerSlashing, flags: UpdateFlags): Result[void, cstring] = @@ -198,7 +198,7 @@ func is_slashable_attestation_data( # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#attester-slashings proc check_attester_slashing*( - state: var ForkyBeaconState, + state: ForkyBeaconState, attester_slashing: SomeAttesterSlashing, flags: UpdateFlags ): Result[seq[ValidatorIndex], cstring] = @@ -255,6 +255,17 @@ proc process_attester_slashing*( ok() +proc findValidatorIndex*(state: ForkyBeaconState, pubkey: ValidatorPubKey): int = + # This linear scan is unfortunate, but should be fairly fast as we do a simple + # byte comparison of the key. The alternative would be to build a Table, but + # given that each block can hold no more than 16 deposits, it's slower to + # build the table and use it for lookups than to scan it like this. + # Once we have a reusable, long-lived cache, this should be revisited + for i in 0 ..< state.validators.len: + if state.validators.asSeq[i].pubkey == pubkey: + return i + return -1 + proc process_deposit*(cfg: RuntimeConfig, state: var ForkyBeaconState, deposit: Deposit, @@ -277,18 +288,7 @@ proc process_deposit*(cfg: RuntimeConfig, let pubkey = deposit.data.pubkey amount = deposit.data.amount - - var index = -1 - - # This linear scan is unfortunate, but should be fairly fast as we do a simple - # byte comparison of the key. The alternative would be to build a Table, but - # given that each block can hold no more than 16 deposits, it's slower to - # build the table and use it for lookups than to scan it like this. - # Once we have a reusable, long-lived cache, this should be revisited - for i in 0.. MIN_EPOCHS_TO_INACTIVITY_PENALTY -func get_finality_delay(state: ForkyBeaconState): uint64 = +func get_finality_delay*(state: ForkyBeaconState): uint64 = get_previous_epoch(state) - state.finalized_checkpoint.epoch # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#rewards-and-penalties-1 @@ -469,6 +469,19 @@ func is_in_inactivity_leak(state: altair.BeaconState | merge.BeaconState): bool # TODO remove this, see above get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY +func get_attestation_component_reward*(attesting_balance: Gwei, + total_balance: Gwei, + base_reward: uint64, + finality_delay: uint64): Gwei = + if is_in_inactivity_leak(finality_delay): + # Since full base reward will be canceled out by inactivity penalty deltas, + # optimal participation receives full base reward compensation here. + base_reward + else: + let reward_numerator = + base_reward * (attesting_balance div EFFECTIVE_BALANCE_INCREMENT) + reward_numerator div (total_balance div EFFECTIVE_BALANCE_INCREMENT) + func get_attestation_component_delta(is_unslashed_attester: bool, attesting_balance: Gwei, total_balance: Gwei, @@ -477,15 +490,11 @@ func get_attestation_component_delta(is_unslashed_attester: bool, # Helper with shared logic for use by get source, target, and head deltas # functions if is_unslashed_attester: - if is_in_inactivity_leak(finality_delay): - # Since full base reward will be canceled out by inactivity penalty deltas, - # optimal participation receives full base reward compensation here. - RewardDelta(rewards: base_reward) - else: - let reward_numerator = - base_reward * (attesting_balance div EFFECTIVE_BALANCE_INCREMENT) - RewardDelta(rewards: - reward_numerator div (total_balance div EFFECTIVE_BALANCE_INCREMENT)) + RewardDelta(rewards: get_attestation_component_reward( + attesting_balance, + total_balance, + base_reward, + finality_delay)) else: RewardDelta(penalties: base_reward) @@ -612,7 +621,7 @@ func get_attestation_deltas(state: phase0.BeaconState, info: var phase0.EpochInf proposer_delta.get()[1]) # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#get_base_reward -func get_base_reward_increment( +func get_base_reward_increment*( state: altair.BeaconState | merge.BeaconState, index: ValidatorIndex, base_reward_per_increment: Gwei): Gwei = ## Return the base reward for the validator defined by ``index`` with respect @@ -621,6 +630,27 @@ func get_base_reward_increment( state.validators[index].effective_balance div EFFECTIVE_BALANCE_INCREMENT increments * base_reward_per_increment +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#get_flag_index_deltas +func get_flag_index_reward*(state: altair.BeaconState | merge.BeaconState, + base_reward: Gwei, active_increments: Gwei, + unslashed_participating_increments: Gwei, + weight: uint64): Gwei = + if not is_in_inactivity_leak(state): + let reward_numerator = + base_reward * weight * unslashed_participating_increments + reward_numerator div (active_increments * WEIGHT_DENOMINATOR) + else: + 0.Gwei + +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#get_flag_index_deltas +func get_unslashed_participating_increment*( + info: altair.EpochInfo, flag_index: int): Gwei = + info.balances.previous_epoch[flag_index] div EFFECTIVE_BALANCE_INCREMENT + +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#get_flag_index_deltas +func get_active_increments*(info: altair.EpochInfo): Gwei = + info.balances.current_epoch div EFFECTIVE_BALANCE_INCREMENT + # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#get_flag_index_deltas iterator get_flag_index_deltas*( state: altair.BeaconState | merge.BeaconState, flag_index: int, @@ -632,12 +662,9 @@ iterator get_flag_index_deltas*( let previous_epoch = get_previous_epoch(state) weight = PARTICIPATION_FLAG_WEIGHTS[flag_index].uint64 # safe - unslashed_participating_balance = - info.balances.previous_epoch[flag_index] - unslashed_participating_increments = - unslashed_participating_balance div EFFECTIVE_BALANCE_INCREMENT - active_increments = - info.balances.current_epoch div EFFECTIVE_BALANCE_INCREMENT + unslashed_participating_increments = get_unslashed_participating_increment( + info, flag_index) + active_increments = get_active_increments(info) for index in 0 ..< state.validators.len: if not is_eligible_validator(info.validators[index]): @@ -657,14 +684,11 @@ iterator get_flag_index_deltas*( info.validators[vidx].flags.incl pflag - if not is_in_inactivity_leak(state): - let reward_numerator = - base_reward * weight * unslashed_participating_increments - (vidx, RewardDelta( - rewards: reward_numerator div (active_increments * WEIGHT_DENOMINATOR), - penalties: 0.Gwei)) - else: - (vidx, RewardDelta(rewards: 0.Gwei, penalties: 0.Gwei)) + (vidx, RewardDelta( + rewards: get_flag_index_reward( + state, base_reward, active_increments, + unslashed_participating_increments, weight), + penalties: 0.Gwei)) elif flag_index != TIMELY_HEAD_FLAG_INDEX: (vidx, RewardDelta( rewards: 0.Gwei, @@ -827,34 +851,55 @@ func process_registry_updates*( # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slashings # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#slashings -# https://github.com/ethereum/consensus-specs/blob/v1.1.5/specs/merge/beacon-chain.md#slashings +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#slashings +func get_adjusted_total_slashing_balance*( + state: ForkyBeaconState, total_balance: Gwei): Gwei = + let multiplier = + # tradeoff here about interleaving phase0/altair, but for these + # single-constant changes... + uint64(when state is phase0.BeaconState: + PROPORTIONAL_SLASHING_MULTIPLIER + elif state is altair.BeaconState: + PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR + elif state is merge.BeaconState: + PROPORTIONAL_SLASHING_MULTIPLIER_MERGE + else: + raiseAssert "process_slashings: incorrect BeaconState type") + return min(sum(state.slashings.data) * multiplier, total_balance) + +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slashings +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#slashings +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#slashings +func slashing_penalty_applies*(validator: Validator, epoch: Epoch): bool = + return validator.slashed and + epoch + EPOCHS_PER_SLASHINGS_VECTOR div 2 == validator.withdrawable_epoch + +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slashings +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#slashings +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#slashings +func get_slashing_penalty*(validator: Validator, + adjusted_total_slashing_balance, + total_balance: Gwei): Gwei = + const increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty + # numerator to avoid uint64 overflow + let penalty_numerator = validator.effective_balance div increment * + adjusted_total_slashing_balance + return penalty_numerator div total_balance * increment + +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slashings +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#slashings +# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#slashings func process_slashings*(state: var ForkyBeaconState, total_balance: Gwei) = let epoch = get_current_epoch(state) - multiplier = - # tradeoff here about interleaving phase0/altair, but for these - # single-constant changes... - uint64(when state is phase0.BeaconState: - PROPORTIONAL_SLASHING_MULTIPLIER - elif state is altair.BeaconState: - PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR - elif state is merge.BeaconState: - PROPORTIONAL_SLASHING_MULTIPLIER_MERGE - else: - raiseAssert "process_slashings: incorrect BeaconState type") - adjusted_total_slashing_balance = - min(sum(state.slashings.data) * multiplier, total_balance) + adjusted_total_slashing_balance = get_adjusted_total_slashing_balance( + state, total_balance) for index in 0.. 0, "Must select at least one block" - for b in 0.. BeaconStateFork.Phase0: + template flags: untyped = auxiliaryState.epochParticipationFlags + flags.currentEpochParticipation = state.data.current_epoch_participation + flags.previousEpochParticipation = state.data.previous_epoch_participation + +proc isPerfect(info: RewardsAndPenalties): bool = + info.slashing_outcome >= 0 and + info.source_outcome == info.max_source_reward.int64 and + info.target_outcome == info.max_target_reward.int64 and + info.head_outcome == info.max_head_reward.int64 and + info.inclusion_delay_outcome == info.max_inclusion_delay_reward.int64 and + info.sync_committee_outcome == info.max_sync_committee_reward.int64 + +proc getMaxEpochFromDbTable(db: SqStoreRef, tableName: string): int64 = + var queryResult: int64 + discard db.exec(&"SELECT MAX(epoch) FROM {tableName}", ()) do (res: int64): + queryResult = res + return queryResult + +proc collectBalances(balances: var seq[uint64], forkedState: ForkedHashedBeaconState) = + withState(forkedState): + balances = seq[uint64](state.data.balances.data) + +proc calculateDelta(info: RewardsAndPenalties): int64 = + info.source_outcome + + info.target_outcome + + info.head_outcome + + info.inclusion_delay_outcome + + info.sync_committee_outcome + + info.proposer_outcome + + info.slashing_outcome - + info.inactivity_penalty.int64 + + info.deposits.int64 + +proc printComponents(info: RewardsAndPenalties) = + echo "Components:" + echo "Source outcome: ", info.source_outcome + echo "Target outcome: ", info.target_outcome + echo "Head outcome: ", info.head_outcome + echo "Inclusion delay outcome: ", info.inclusion_delay_outcome + echo "Sync committee outcome: ", info.sync_committee_outcome + echo "Proposer outcome: ", info.proposer_outcome + echo "Slashing outcome: ", info.slashing_outcome + echo "Inactivity penalty: ", info.inactivity_penalty + echo "Deposits: ", info.deposits + +proc checkBalance(validatorIndex: int64, + validator: RewardStatus | ParticipationInfo, + currentEpochBalance, previousEpochBalance: Gwei, + validatorInfo: RewardsAndPenalties) = + let delta = validatorInfo.calculateDelta + if currentEpochBalance.int64 == previousEpochBalance.int64 + delta: + return + echo "Validator: ", validatorIndex + echo "Is eligible: ", is_eligible_validator(validator) + echo "Current epoch balance: ", currentEpochBalance + echo "Previous epoch balance: ", previousEpochBalance + echo "State delta: ", currentEpochBalance - previousEpochBalance + echo "Computed delta: ", delta + printComponents(validatorInfo) + raiseAssert("Validator's previous epoch balance plus computed validator's " & + "delta is not equal to the validator's current epoch balance.") + +proc getDbValidatorsCount(db: SqStoreRef): int64 = + var res: int64 + discard db.exec("SELECT count(*) FROM validators", ()) do (r: int64): + res = r + return res + +template inTransaction(db: SqStoreRef, dbName: string, body: untyped) = + try: + db.exec("BEGIN TRANSACTION;").expect(dbName) + body + finally: + db.exec("END TRANSACTION;").expect(dbName) + +proc insertValidators(db: SqStoreRef, state: ForkedHashedBeaconState, + startIndex, endIndex: int64) = + var insertValidator {.global.}: SqliteStmt[ + (int64, array[48, byte], array[32, byte]), void] + once: insertValidator = db.createInsertValidatorProc + withState(state): + db.inTransaction("DB"): + for i in startIndex ..< endIndex: + insertValidator.exec((i, state.data.validators[i].pubkey.toRaw, + state.data.validators[i].withdrawal_credentials.data)).expect("DB") + +proc getOutcome(delta: RewardDelta): int64 = + delta.rewards.int64 - delta.penalties.int64 + +proc collectSlashings( + rewardsAndPenalties: var seq[RewardsAndPenalties], + state: ForkyBeaconState, total_balance: Gwei) = let - insertValidator = outDb.prepareStmt(""" - INSERT INTO validators_raw( - validator_index, - pubkey, - withdrawal_credentials) - VALUES(?, ?, ?);""", - (int64, array[48, byte], array[32, byte]), void).expect("DB") - insertEpochInfo = outDb.prepareStmt(""" - INSERT INTO epoch_info( - epoch, - current_epoch_raw, - previous_epoch_raw, - current_epoch_attesters_raw, - current_epoch_target_attesters_raw, - previous_epoch_attesters_raw, - previous_epoch_target_attesters_raw, - previous_epoch_head_attesters_raw) - VALUES(?, ?, ?, ?, ?, ?, ?, ?);""", - (int64, int64, int64, int64, int64, int64, int64, int64), void).expect("DB") - insertValidatorInfo = outDb.prepareStmt(""" - INSERT INTO validator_epoch_info( - validator_index, - epoch, - rewards, - penalties, - source_attester, - target_attester, - head_attester, - inclusion_delay) - VALUES(?, ?, ?, ?, ?, ?, ?, ?);""", - (int64, int64, int64, int64, int64, int64, int64, Option[int64]), void).expect("DB") - - var vals: int64 - discard outDb.exec("SELECT count(*) FROM validators", ()) do (res: int64): - vals = res - - outDb.exec("BEGIN TRANSACTION;").expect("DB") - - for i in vals.. BeaconStateFork.Phase0: + let base_reward_per_increment = get_base_reward_per_increment( + get_total_active_balance(state.data, cache)) + doAssert base_reward_per_increment > 0 + for attestation in blck.message.body.attestations: + doAssert check_attestation(state.data, attestation, {}, cache).isOk + let proposerReward = + if attestation.data.target.epoch == get_current_epoch(state.data): + get_proposer_reward( + state.data, attestation, base_reward_per_increment, cache, + epochParticipationFlags.currentEpochParticipation) + else: + get_proposer_reward( + state.data, attestation, base_reward_per_increment, cache, + epochParticipationFlags.previousEpochParticipation) + rewardsAndPenalties[blck.message.proposer_index].proposer_outcome += + proposerReward.int64 + let inclusionDelay = state.data.slot - attestation.data.slot + for index in get_attesting_indices( + state.data, attestation.data, attestation.aggregation_bits, cache): + rewardsAndPenalties[index].inclusion_delay = some(inclusionDelay.int64) + +proc collectFromDeposits( + rewardsAndPenalties: var seq[RewardsAndPenalties], + forkedState: ForkedHashedBeaconState, + forkedBlock: ForkedTrustedSignedBeaconBlock, + pubkeyToIndex: var PubkeyToIndexTable, + cfg: RuntimeConfig) = + withStateAndBlck(forkedState, forkedBlock): + for deposit in blck.message.body.deposits: + let pubkey = deposit.data.pubkey + let amount = deposit.data.amount + var index = findValidatorIndex(state.data, pubkey) + if index == -1: + index = pubkeyToIndex.getOrDefault(pubkey, -1) + if index != -1: + rewardsAndPenalties[index].deposits += amount + elif verify_deposit_signature(cfg, deposit.data): + pubkeyToIndex[pubkey] = rewardsAndPenalties.len + rewardsAndPenalties.add( + RewardsAndPenalties(deposits: amount)) + +proc collectFromSyncAggregate( + rewardsAndPenalties: var seq[RewardsAndPenalties], + forkedState: ForkedHashedBeaconState, + forkedBlock: ForkedTrustedSignedBeaconBlock, + cache: var StateCache) = + withStateAndBlck(forkedState, forkedBlock): + when stateFork > BeaconStateFork.Phase0: + let total_active_balance = get_total_active_balance(state.data, cache) + let participant_reward = get_participant_reward(total_active_balance) + let proposer_reward = + state_transition_block.get_proposer_reward(participant_reward) + let indices = get_sync_committee_cache(state.data, cache).current_sync_committee + + template aggregate: untyped = blck.message.body.sync_aggregate + + doAssert indices.len == SYNC_COMMITTEE_SIZE + doAssert aggregate.sync_committee_bits.len == SYNC_COMMITTEE_SIZE + doAssert state.data.current_sync_committee.pubkeys.len == SYNC_COMMITTEE_SIZE + + for i in 0 ..< SYNC_COMMITTEE_SIZE: + rewardsAndPenalties[indices[i]].max_sync_committee_reward += + participant_reward + if aggregate.sync_committee_bits[i]: + rewardsAndPenalties[indices[i]].sync_committee_outcome += + participant_reward.int64 + rewardsAndPenalties[blck.message.proposer_index].proposer_outcome += + proposer_reward.int64 + else: + rewardsAndPenalties[indices[i]].sync_committee_outcome -= + participant_reward.int64 + +proc collectBlockRewardsAndPenalties( + rewardsAndPenalties: var seq[RewardsAndPenalties], + forkedState: ForkedHashedBeaconState, + forkedBlock: ForkedTrustedSignedBeaconBlock, + auxiliaryState: var AuxiliaryState, + cache: var StateCache, cfg: RuntimeConfig) = + rewardsAndPenalties.collectFromProposerSlashings(forkedState, forkedBlock) + rewardsAndPenalties.collectFromAttesterSlashings(forkedState, forkedBlock) + rewardsAndPenalties.collectFromAttestations( + forkedState, forkedBlock, auxiliaryState.epochParticipationFlags, cache) + rewardsAndPenalties.collectFromDeposits( + forkedState, forkedBlock, auxiliaryState.pubkeyToIndex, cfg) + # This table is needed only to resolve double deposits in the same block, so + # it can be cleared after processing all deposits for the current block. + auxiliaryState.pubkeyToIndex.clear + rewardsAndPenalties.collectFromSyncAggregate(forkedState, forkedBlock, cache) + +proc cmdValidatorDb(conf: DbConf, cfg: RuntimeConfig) = + # Create a database with performance information for every epoch + echo "Opening database..." + let db = BeaconChainDB.new(conf.databaseDir.string, false, true) + defer: db.close() + + if (let v = ChainDAGRef.isInitialized(db); v.isErr()): + echo "Database not initialized" + quit 1 + + echo "Initializing block pool..." + let + validatorMonitor = newClone(ValidatorMonitor.init()) + dag = ChainDAGRef.init(cfg, db, validatorMonitor, {}) + + let outDb = SqStoreRef.init(conf.outDir, "validatorDb").expect("DB") + defer: outDb.close() + + outDb.createValidatorsRawTable + outDb.createValidatorsView + outDb.createPhase0EpochInfoTable + outDb.createAltairEpochInfoTable + outDb.createValidatorEpochInfoTable let + insertPhase0EpochInfo = outDb.createInsertPhase0EpochInfoProc + insertAltairEpochInfo = outDb.createInsertAltairEpochInfoProc + insertValidatorInfo = outDb.createInsertValidatorEpochInfoProc + minEpoch = + if conf.startEpoch == 0: + Epoch(max(outDb.getMaxEpochFromDbTable("phase0_epoch_info"), + outDb.getMaxEpochFromDbTable("altair_epoch_info")) + 1) + else: + Epoch(conf.startEpoch) start = minEpoch.compute_start_slot_at_epoch() ends = dag.finalizedHead.slot # Avoid dealing with changes @@ -830,22 +1256,80 @@ proc cmdValidatorDb(conf: DbConf, cfg: RuntimeConfig) = echo "Analyzing performance for epochs ", start.epoch, " - ", ends.epoch - let state = newClone(dag.headState) - dag.updateStateData( - state[], blockRefs[^1].atSlot(if start > 0: start - 1 else: 0.Slot), - false, cache) + let tmpState = newClone(dag.headState) + var cache = StateCache() + let slot = if start > 0: start - 1 else: 0.Slot + if blockRefs.len > 0: + dag.updateStateData(tmpState[], blockRefs[^1].atSlot(slot), false, cache) + else: + dag.updateStateData(tmpState[], dag.head.atSlot(slot), false, cache) + + let dbValidatorsCount = outDb.getDbValidatorsCount() + var validatorsCount = getStateField(tmpState[].data, validators).len + outDb.insertValidators(tmpState[].data, dbValidatorsCount, validatorsCount) + + var previousEpochBalances: seq[uint64] + collectBalances(previousEpochBalances, tmpState[].data) + + var forkedInfo = ForkedEpochInfo() + var rewardsAndPenalties: seq[RewardsAndPenalties] + rewardsAndPenalties.setLen(validatorsCount) + + var auxiliaryState: AuxiliaryState + auxiliaryState.copyParticipationFlags(tmpState[].data) - var inTxn = false proc processEpoch() = - echo getStateField(state[].data, slot).epoch - if not inTxn: - outDb.exec("BEGIN TRANSACTION;").expect("DB") - inTxn = true - case info.kind + let epoch = getStateField(tmpState[].data, slot).epoch.int64 + echo epoch + + withState(tmpState[].data): + withEpochInfo(forkedInfo): + doAssert state.data.balances.len == info.validators.len + doAssert state.data.balances.len == previousEpochBalances.len + doAssert state.data.balances.len == rewardsAndPenalties.len + + for index, validator in info.validators.pairs: + template outputInfo: untyped = rewardsAndPenalties[index] + + checkBalance(index, validator, state.data.balances[index], + previousEpochBalances[index], outputInfo) + + let delay = + when infoFork == EpochInfoFork.Phase0: + let notSlashed = (RewardFlags.isSlashed notin validator.flags) + if notSlashed and validator.is_previous_epoch_attester.isSome(): + some(int64(validator.is_previous_epoch_attester.get().delay)) + else: + none(int64) + else: + rewardsAndPenalties[index].inclusion_delay + + if conf.perfect or not outputInfo.isPerfect: + insertValidatorInfo.exec(( + index.int64, + epoch, + outputInfo.source_outcome, + outputInfo.max_source_reward.int64, + outputInfo.target_outcome, + outputInfo.max_target_reward.int64, + outputInfo.head_outcome, + outputInfo.max_head_reward.int64, + outputInfo.inclusion_delay_outcome, + outputInfo.max_inclusion_delay_reward.int64, + outputInfo.sync_committee_outcome, + outputInfo.max_sync_committee_reward.int64, + outputInfo.proposer_outcome, + outputInfo.inactivity_penalty.int64, + outputInfo.slashing_outcome, + delay)).expect("DB") + + collectBalances(previousEpochBalances, tmpState[].data) + + case forkedInfo.kind of EpochInfoFork.Phase0: - template info: untyped = info.phase0Data - insertEpochInfo.exec( - (getStateField(state[].data, slot).epoch.int64, + template info: untyped = forkedInfo.phase0Data + insertPhase0EpochInfo.exec(( + epoch, info.balances.current_epoch_raw.int64, info.balances.previous_epoch_raw.int64, info.balances.current_epoch_attesters_raw.int64, @@ -854,77 +1338,65 @@ proc cmdValidatorDb(conf: DbConf, cfg: RuntimeConfig) = info.balances.previous_epoch_target_attesters_raw.int64, info.balances.previous_epoch_head_attesters_raw.int64) ).expect("DB") - - for index, status in info.validators.pairs(): - if not is_eligible_validator(status): - continue - let - notSlashed = (RewardFlags.isSlashed notin status.flags) - source_attester = - notSlashed and status.is_previous_epoch_attester.isSome() - target_attester = - notSlashed and RewardFlags.isPreviousEpochTargetAttester in status.flags - head_attester = - notSlashed and RewardFlags.isPreviousEpochHeadAttester in status.flags - delay = - if notSlashed and status.is_previous_epoch_attester.isSome(): - some(int64(status.is_previous_epoch_attester.get().delay)) - else: - none(int64) - - if conf.perfect or not - (source_attester and target_attester and head_attester and - delay.isSome() and delay.get() == 1): - insertValidatorInfo.exec( - (index.int64, - getStateField(state[].data, slot).epoch.int64, - status.delta.rewards.int64, - status.delta.penalties.int64, - int64(source_attester), # Source delta - int64(target_attester), # Target delta - int64(head_attester), # Head delta - delay)).expect("DB") of EpochInfoFork.Altair: - echo "TODO altair support" - - if getStateField(state[].data, slot).epoch.int64 mod 16 == 0: - inTxn = false - outDb.exec("COMMIT;").expect("DB") - - for bi in 0.. validatorsCount: + # Resize the structures in case a new validator has appeared after + # the state_transition_block procedure call ... + rewardsAndPenalties.setLen(newValidatorsCount) + previousEpochBalances.setLen(newValidatorsCount) + # ... and add the new validators to the database. + outDb.insertValidators(tmpState[].data, validatorsCount, newValidatorsCount) + validatorsCount = newValidatorsCount # Capture rewards of empty slots as well, including the epoch that got # finalized - while getStateField(state[].data, slot) <= ends: - let ok = process_slots( - cfg, state[].data, getStateField(state[].data, slot) + 1, cache, - info, {}) - doAssert ok, "Slot processing can't fail with correct inputs" - - if getStateField(state[].data, slot).isEpoch(): - processEpoch() - - if inTxn: - inTxn = false - outDb.exec("COMMIT;").expect("DB") + processSlots(ends, {}) when isMainModule: var