Precise per-component ETH-denominated rewards tracking

This is an alternative take on #3107
that aims for more minimal interventions in the spec modules at the expense of
duplicating more of the spec logic in ncli_db.

beacon_chain/beacon_chain_db.nim

@@ -76,7 +76,7 @@ type
     ## database - this may have a number of "natural" causes such as switching
     ## between different versions of the client and accidentally using an old
     ## database.
-    db: SqStoreRef
+    db*: SqStoreRef
 
     v0: BeaconChainDBV0
     genesisDeposits*: DepositsSeq
@@ -269,25 +269,27 @@ proc loadImmutableValidators(vals: DbSeq[ImmutableValidatorDataDb2]): seq[Immuta
 proc new*(T: type BeaconChainDB,
           dir: string,
           inMemory = false,
+          readOnly = false
     ): BeaconChainDB =
   var db = if inMemory:
-      SqStoreRef.init("", "test", inMemory = true).expect(
+      SqStoreRef.init("", "test", readOnly = readOnly, inMemory = true).expect(
         "working database (out of memory?)")
     else:
       let s = secureCreatePath(dir)
       doAssert s.isOk # TODO(zah) Handle this in a better way
 
       SqStoreRef.init(
-        dir, "nbc", manualCheckpoint = true).expectDb()
+        dir, "nbc", readOnly = readOnly, manualCheckpoint = true).expectDb()
 
-  # Remove the deposits table we used before we switched
-  # to storing only deposit contract checkpoints
-  if db.exec("DROP TABLE IF EXISTS deposits;").isErr:
-    debug "Failed to drop the deposits table"
+  if not readOnly:
+    # Remove the deposits table we used before we switched
+    # to storing only deposit contract checkpoints
+    if db.exec("DROP TABLE IF EXISTS deposits;").isErr:
+      debug "Failed to drop the deposits table"
 
-  # An old pubkey->index mapping that hasn't been used on any mainnet release
-  if db.exec("DROP TABLE IF EXISTS validatorIndexFromPubKey;").isErr:
-    debug "Failed to drop the validatorIndexFromPubKey table"
+    # An old pubkey->index mapping that hasn't been used on any mainnet release
+    if db.exec("DROP TABLE IF EXISTS validatorIndexFromPubKey;").isErr:
+      debug "Failed to drop the validatorIndexFromPubKey table"
 
   var
     # V0 compatibility tables - these were created WITHOUT ROWID which is slow

beacon_chain/beacon_chain_db_immutable.nim

@@ -107,10 +107,8 @@ type
     ## Per-epoch sums of slashed effective balances
 
     # Participation
-    previous_epoch_participation*:
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
-    current_epoch_participation*:
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
+    previous_epoch_participation*: EpochParticipationFlags
+    current_epoch_participation*: EpochParticipationFlags
 
     # Finality
     justification_bits*: uint8 ##\
@@ -169,10 +167,8 @@ type
     ## Per-epoch sums of slashed effective balances
 
     # Participation
-    previous_epoch_participation*:
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
-    current_epoch_participation*:
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
+    previous_epoch_participation*: EpochParticipationFlags
+    current_epoch_participation*: EpochParticipationFlags
 
     # Finality
     justification_bits*: uint8 ##\

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -546,9 +546,10 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
   # Pruning metadata
   dag.lastPrunePoint = dag.finalizedHead
 
-  # Fill validator key cache in case we're loading an old database that doesn't
-  # have a cache
-  dag.updateValidatorKeys(getStateField(dag.headState.data, validators).asSeq())
+  if not dag.db.db.readOnly:
+    # Fill validator key cache in case we're loading an old database that doesn't
+    # have a cache
+    dag.updateValidatorKeys(getStateField(dag.headState.data, validators).asSeq())
 
   withState(dag.headState.data):
     when stateFork >= BeaconStateFork.Altair:

beacon_chain/spec/beaconstate.nim

@@ -119,6 +119,39 @@ func initiate_validator_exit*(cfg: RuntimeConfig, state: var ForkyBeaconState,
   validator.withdrawable_epoch =
     validator.exit_epoch + cfg.MIN_VALIDATOR_WITHDRAWABILITY_DELAY
 
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slash_validator
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#modified-slash_validator
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#modified-slash_validator
+proc get_slashing_penalty*(state: ForkyBeaconState,
+                           validator_effective_balance: Gwei): Gwei =
+  # TODO Consider whether this is better than splitting the functions apart; in
+  # each case, tradeoffs. Here, it's just changing a couple of constants.
+  when state is phase0.BeaconState:
+      validator_effective_balance div MIN_SLASHING_PENALTY_QUOTIENT
+  elif state is altair.BeaconState:
+      validator_effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR
+  elif state is merge.BeaconState:
+      validator_effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_MERGE
+  else:
+    raiseAssert "invalid BeaconState type"
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slash_validator
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#modified-slash_validator
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#modified-slash_validator
+proc get_whistleblower_reward*(validator_effective_balance: Gwei): Gwei =
+  validator_effective_balance div WHISTLEBLOWER_REWARD_QUOTIENT
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slash_validator
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#modified-slash_validator
+# https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#modified-slash_validator
+proc get_proposer_reward(state: ForkyBeaconState, whistleblower_reward: Gwei): Gwei =
+  when state is phase0.BeaconState:
+    whistleblower_reward div PROPOSER_REWARD_QUOTIENT
+  elif state is altair.BeaconState or state is merge.BeaconState:
+    whistleblower_reward * PROPOSER_WEIGHT div WEIGHT_DENOMINATOR
+  else:
+    raiseAssert "invalid BeaconState type"
+
 # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#slash_validator
 # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#modified-slash_validator
 # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/merge/beacon-chain.md#modified-slash_validator
@@ -145,19 +178,8 @@ proc slash_validator*(
   state.slashings[int(epoch mod EPOCHS_PER_SLASHINGS_VECTOR)] +=
     validator.effective_balance
 
-  # TODO Consider whether this is better than splitting the functions apart; in
-  # each case, tradeoffs. Here, it's just changing a couple of constants.
-  when state is phase0.BeaconState:
-    decrease_balance(state, slashed_index,
-      validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT)
-  elif state is altair.BeaconState:
-    decrease_balance(state, slashed_index,
-      validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR)
-  elif state is merge.BeaconState:
-    decrease_balance(state, slashed_index,
-      validator.effective_balance div MIN_SLASHING_PENALTY_QUOTIENT_MERGE)
-  else:
-    raiseAssert "invalid BeaconState type"
+  decrease_balance(state, slashed_index,
+    get_slashing_penalty(state, validator.effective_balance))
 
   # The rest doesn't make sense without there being any proposer index, so skip
   let proposer_index = get_beacon_proposer_index(state, cache)
@@ -169,15 +191,8 @@ proc slash_validator*(
   let
     # Spec has whistleblower_index as optional param, but it's never used.
     whistleblower_index = proposer_index.get
-    whistleblower_reward =
-      (validator.effective_balance div WHISTLEBLOWER_REWARD_QUOTIENT).Gwei
-    proposer_reward =
-      when state is phase0.BeaconState:
-        whistleblower_reward div PROPOSER_REWARD_QUOTIENT
-      elif state is altair.BeaconState or state is merge.BeaconState:
-        whistleblower_reward * PROPOSER_WEIGHT div WEIGHT_DENOMINATOR
-      else:
-        raiseAssert "invalid BeaconState type"
+    whistleblower_reward = get_whistleblower_reward(validator.effective_balance)
+    proposer_reward = get_proposer_reward(state, whistleblower_reward)
 
   increase_balance(state, proposer_index.get, proposer_reward)
   # TODO: evaluate if spec bug / underflow can be triggered
@@ -623,6 +638,31 @@ proc check_attestation*(
 
   ok()
 
+proc get_proposer_reward*(state: ForkyBeaconState,
+                          attestation: SomeAttestation,
+                          base_reward_per_increment: Gwei,
+                          cache: var StateCache,
+                          epoch_participation: var EpochParticipationFlags): uint64 =
+  let participation_flag_indices = get_attestation_participation_flag_indices(
+    state, attestation.data, state.slot - attestation.data.slot)
+  for index in get_attesting_indices(
+      state, attestation.data, attestation.aggregation_bits, cache):
+    for flag_index, weight in PARTICIPATION_FLAG_WEIGHTS:
+      if flag_index in participation_flag_indices and
+         not has_flag(epoch_participation.asSeq[index], flag_index):
+        epoch_participation.asSeq[index] =
+          add_flag(epoch_participation.asSeq[index], flag_index)
+        # these are all valid; TODO statically verify or do it type-safely
+        result += get_base_reward(
+          state, index, base_reward_per_increment) * weight.uint64
+  epoch_participation.clearCache()
+
+  let proposer_reward_denominator =
+    (WEIGHT_DENOMINATOR.uint64 - PROPOSER_WEIGHT.uint64) *
+    WEIGHT_DENOMINATOR.uint64 div PROPOSER_WEIGHT.uint64
+
+  return result div proposer_reward_denominator
+
 proc process_attestation*(
     state: var ForkyBeaconState, attestation: SomeAttestation, flags: UpdateFlags,
     base_reward_per_increment: Gwei, cache: var StateCache):
@@ -656,31 +696,8 @@ proc process_attestation*(
 
   # Altair and Merge
   template updateParticipationFlags(epoch_participation: untyped) =
-    var proposer_reward_numerator = 0'u64
-
-    # Participation flag indices
-    let participation_flag_indices =
-      get_attestation_participation_flag_indices(
-        state, attestation.data, state.slot - attestation.data.slot)
-
-    for index in get_attesting_indices(
-        state, attestation.data, attestation.aggregation_bits, cache):
-      for flag_index, weight in PARTICIPATION_FLAG_WEIGHTS:
-        if flag_index in participation_flag_indices and
-            not has_flag(epoch_participation.asSeq[index], flag_index):
-          epoch_participation.asSeq[index] =
-            add_flag(epoch_participation.asSeq[index], flag_index)
-
-          # these are all valid; TODO statically verify or do it type-safely
-          proposer_reward_numerator += get_base_reward(
-            state, index, base_reward_per_increment) * weight.uint64
-    epoch_participation.clearCache()
-
-    # Reward proposer
-    let
-      # TODO use correct type at source
-      proposer_reward_denominator = (WEIGHT_DENOMINATOR.uint64 - PROPOSER_WEIGHT.uint64) * WEIGHT_DENOMINATOR.uint64 div PROPOSER_WEIGHT.uint64
-      proposer_reward = Gwei(proposer_reward_numerator div proposer_reward_denominator)
+    let proposer_reward = get_proposer_reward(
+      state, attestation, base_reward_per_increment, cache, epoch_participation)
     increase_balance(state, proposer_index.get, proposer_reward)
 
   when state is phase0.BeaconState:
@@ -777,8 +794,7 @@ func translate_participation(
 
 proc upgrade_to_altair*(cfg: RuntimeConfig, pre: phase0.BeaconState): ref altair.BeaconState =
   var
-    empty_participation =
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]()
+    empty_participation = EpochParticipationFlags()
     inactivity_scores = HashList[uint64, Limit VALIDATOR_REGISTRY_LIMIT]()
 
   doAssert empty_participation.data.setLen(pre.validators.len)

beacon_chain/spec/datatypes/altair.nim

@@ -78,6 +78,9 @@ type
   # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#custom-types
   ParticipationFlags* = uint8
 
+  EpochParticipationFlags* =
+    HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
+
   # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/altair/beacon-chain.md#syncaggregate
   SyncAggregate* = object
     sync_committee_bits*: BitArray[SYNC_COMMITTEE_SIZE]
@@ -218,10 +221,8 @@ type
     ## Per-epoch sums of slashed effective balances
 
     # Participation
-    previous_epoch_participation*:
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
-    current_epoch_participation*:
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
+    previous_epoch_participation*: EpochParticipationFlags
+    current_epoch_participation*: EpochParticipationFlags
 
     # Finality
     justification_bits*: uint8 ##\

beacon_chain/spec/datatypes/merge.nim

@@ -125,10 +125,8 @@ type
     ## Per-epoch sums of slashed effective balances
 
     # Participation
-    previous_epoch_participation*:
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
-    current_epoch_participation*:
-      HashList[ParticipationFlags, Limit VALIDATOR_REGISTRY_LIMIT]
+    previous_epoch_participation*: EpochParticipationFlags
+    current_epoch_participation*: EpochParticipationFlags
 
     # Finality
     justification_bits*: uint8 ##\

beacon_chain/spec/forks.nim

@@ -218,9 +218,11 @@ template withState*(x: ForkedHashedBeaconState, body: untyped): untyped =
 template withEpochInfo*(x: ForkedEpochInfo, body: untyped): untyped =
   case x.kind
   of EpochInfoFork.Phase0:
+    const infoFork {.inject.} = EpochInfoFork.Phase0
     template info: untyped {.inject.} = x.phase0Data
     body
   of EpochInfoFork.Altair:
+    const infoFork {.inject.} = EpochInfoFork.Altair
     template info: untyped {.inject.} = x.altairData
     body
 

beacon_chain/spec/state_transition_block.nim

@@ -124,7 +124,7 @@ func is_slashable_validator(validator: Validator, epoch: Epoch): bool =
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#proposer-slashings
 proc check_proposer_slashing*(
-    state: var ForkyBeaconState, proposer_slashing: SomeProposerSlashing,
+    state: ForkyBeaconState, proposer_slashing: SomeProposerSlashing,
     flags: UpdateFlags):
     Result[void, cstring] =
 
@@ -198,7 +198,7 @@ func is_slashable_attestation_data(
 
 # https://github.com/ethereum/consensus-specs/blob/v1.1.6/specs/phase0/beacon-chain.md#attester-slashings
 proc check_attester_slashing*(
-       state: var ForkyBeaconState,
+       state: ForkyBeaconState,
        attester_slashing: SomeAttesterSlashing,
        flags: UpdateFlags
      ): Result[seq[ValidatorIndex], cstring] =
@@ -255,6 +255,17 @@ proc process_attester_slashing*(
 
   ok()
 
+proc findValidatorIndex*(state: ForkyBeaconState, pubkey: ValidatorPubKey): int =
+  # This linear scan is unfortunate, but should be fairly fast as we do a simple
+  # byte comparison of the key. The alternative would be to build a Table, but
+  # given that each block can hold no more than 16 deposits, it's slower to
+  # build the table and use it for lookups than to scan it like this.
+  # Once we have a reusable, long-lived cache, this should be revisited
+  for i in 0 ..< state.validators.len:
+    if state.validators.asSeq[i].pubkey == pubkey:
+      return i
+  return -1
+
 proc process_deposit*(cfg: RuntimeConfig,
                       state: var ForkyBeaconState,
                       deposit: Deposit,
@@ -277,18 +288,7 @@ proc process_deposit*(cfg: RuntimeConfig,
   let
     pubkey = deposit.data.pubkey
     amount = deposit.data.amount
-
-  var index = -1
-
-  # This linear scan is unfortunate, but should be fairly fast as we do a simple
-  # byte comparison of the key. The alternative would be to build a Table, but
-  # given that each block can hold no more than 16 deposits, it's slower to
-  # build the table and use it for lookups than to scan it like this.
-  # Once we have a reusable, long-lived cache, this should be revisited
-  for i in 0..<state.validators.len():
-    if state.validators.asSeq()[i].pubkey == pubkey:
-      index = i
-      break
+    index = findValidatorIndex(state, pubkey)
 
   if index != -1:
     # Increase balance by deposit amount
@@ -425,6 +425,21 @@ proc process_operations(cfg: RuntimeConfig,
 
   ok()
 
+# https://github.com/ethereum/consensus-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#sync-committee-processing
+func get_participant_reward*(total_active_balance: Gwei): Gwei =
+  let
+    total_active_increments =
+      total_active_balance div EFFECTIVE_BALANCE_INCREMENT
+    total_base_rewards =
+      get_base_reward_per_increment(total_active_balance) * total_active_increments
+    max_participant_rewards =
+      total_base_rewards * SYNC_REWARD_WEIGHT div WEIGHT_DENOMINATOR div SLOTS_PER_EPOCH
+  return max_participant_rewards div SYNC_COMMITTEE_SIZE
+
+# https://github.com/ethereum/consensus-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#sync-committee-processing
+func get_proposer_reward*(participant_reward: Gwei): Gwei =
+  participant_reward * PROPOSER_WEIGHT div (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)
+
 # https://github.com/ethereum/consensus-specs/blob/v1.1.0-alpha.6/specs/altair/beacon-chain.md#sync-committee-processing
 proc process_sync_aggregate*(
     state: var (altair.BeaconState | merge.BeaconState),
@@ -457,15 +472,8 @@ proc process_sync_aggregate*(
 
   # Compute participant and proposer rewards
   let
-    total_active_increments =
-      total_active_balance div EFFECTIVE_BALANCE_INCREMENT
-    total_base_rewards =
-      get_base_reward_per_increment(total_active_balance) * total_active_increments
-    max_participant_rewards =
-      total_base_rewards * SYNC_REWARD_WEIGHT div WEIGHT_DENOMINATOR div SLOTS_PER_EPOCH
-    participant_reward = max_participant_rewards div SYNC_COMMITTEE_SIZE
-    proposer_reward =
-      participant_reward * PROPOSER_WEIGHT div (WEIGHT_DENOMINATOR - PROPOSER_WEIGHT)
+    participant_reward = get_participant_reward(total_active_balance)
+    proposer_reward = state_transition_block.get_proposer_reward(participant_reward)
     proposer_index = get_beacon_proposer_index(state, cache)
 
   if proposer_index.isNone: