restore EpochRef and flush statecaches on epoch transitions (#1312)

* restore EpochRef and flush statecaches on epoch transitions

* more targeted cache invalidation

* remove get_empty_per_epoch_cache(); implement simpler but still faster get_beacon_proposer_index()/compute_proposer_index() approach; add some abstraction layer for accessing the shuffled validator indices cache

* reduce integer type conversions

* remove most of rest of integer type conversion in compute_proposer_index()

beacon_chain/attestation_aggregation.nim

@@ -48,7 +48,7 @@ proc aggregate_attestations*(
   # TODO for testing purposes, refactor this into the condition check
   # and just calculation
   # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#aggregation-selection
-  var cache = get_empty_per_epoch_cache()
+  var cache = StateCache()
   if not is_aggregator(state, slot, index, slot_signature, cache):
     return none(AggregateAndProof)
 

beacon_chain/attestation_pool.nim

@@ -13,7 +13,7 @@ import
   # Status libraries
   chronicles, stew/[byteutils], json_serialization/std/sets,
   # Internal
-  ./spec/[beaconstate, datatypes, crypto, digest, helpers, validator],
+  ./spec/[beaconstate, datatypes, crypto, digest, helpers],
   ./extras, ./block_pool, ./block_pools/candidate_chains, ./beacon_node_types,
   ./fork_choice/fork_choice
 
@@ -464,7 +464,7 @@ proc getAttestationsForBlock*(pool: AttestationPool,
   if attestations.len == 0:
     return
 
-  var cache = get_empty_per_epoch_cache()
+  var cache = StateCache()
   for a in attestations:
     var
       # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#construct-attestation

beacon_chain/block_pools/candidate_chains.nim

@@ -14,7 +14,9 @@ import
   metrics,
   # Internals
   ../ssz/merkleization, ../beacon_chain_db, ../extras,
-  ../spec/[crypto, datatypes, digest, helpers, validator, state_transition],
+  ../spec/[
+    crypto, datatypes, digest, helpers, validator, state_transition,
+    beaconstate],
   block_pools_types
 
 declareCounter beacon_reorgs_total, "Total occurrences of reorganizations of the chain" # On fork choice
@@ -58,7 +60,7 @@ func parent*(bs: BlockSlot): BlockSlot =
     )
 
 func populateEpochCache(state: BeaconState, epoch: Epoch): EpochRef =
-  result = (EpochRef)(
+  (EpochRef)(
     epoch: state.slot.compute_epoch_at_slot,
     shuffled_active_validator_indices:
       get_shuffled_active_validator_indices(state, epoch))
@@ -167,7 +169,12 @@ func getEpochInfo*(blck: BlockRef, state: BeaconState): EpochRef =
 
   if matching_epochinfo.len == 0:
     let cache = populateEpochCache(state, state_epoch)
-    blck.epochsInfo.add(cache)
+
+    # Don't use BlockRef caching as far as the epoch where the active
+    # validator indices can diverge.
+    if (compute_activation_exit_epoch(blck.slot.compute_epoch_at_slot) >
+        state_epoch):
+      blck.epochsInfo.add(cache)
     trace "candidate_chains.getEpochInfo: back-filling parent.epochInfo",
       state_slot = state.slot
     cache
@@ -177,13 +184,11 @@ func getEpochInfo*(blck: BlockRef, state: BeaconState): EpochRef =
     raiseAssert "multiple EpochRefs per epoch per BlockRef invalid"
 
 func getEpochCache*(blck: BlockRef, state: BeaconState): StateCache =
-  when false:
-    let epochInfo = getEpochInfo(blck, state)
-    result = get_empty_per_epoch_cache()
-    result.shuffled_active_validator_indices[
-      state.slot.compute_epoch_at_slot] =
-        epochInfo.shuffled_active_validator_indices
-  get_empty_per_epoch_cache()
+  let epochInfo = getEpochInfo(blck, state)
+  result = StateCache()
+  result.shuffled_active_validator_indices[
+    state.slot.compute_epoch_at_slot] =
+      epochInfo.shuffled_active_validator_indices
 
 func init(T: type BlockRef, root: Eth2Digest, slot: Slot): BlockRef =
   BlockRef(
@@ -920,7 +925,7 @@ proc getProposer*(
     dag: CandidateChains, head: BlockRef, slot: Slot):
     Option[(ValidatorIndex, ValidatorPubKey)] =
   dag.withState(dag.tmpState, head.atSlot(slot)):
-    var cache = get_empty_per_epoch_cache()
+    var cache = StateCache()
 
     let proposerIdx = get_beacon_proposer_index(state, cache)
     if proposerIdx.isNone:

beacon_chain/spec/beaconstate.nim

@@ -107,7 +107,7 @@ proc process_deposit*(preset: RuntimePreset,
   ok()
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_activation_exit_epoch
-func compute_activation_exit_epoch(epoch: Epoch): Epoch =
+func compute_activation_exit_epoch*(epoch: Epoch): Epoch =
   ## Return the epoch during which validator activations and exits initiated in
   ## ``epoch`` take effect.
   epoch + 1 + MAX_SEED_LOOKAHEAD
@@ -592,10 +592,13 @@ proc check_attestation*(
   trace "process_attestation: beginning",
     attestation=attestation
 
-  if not (data.index < get_committee_count_at_slot(state, data.slot)):
+  let committee_count_at_slot =
+    get_committee_count_at_slot(get_shuffled_active_validator_indices(
+      state, stateSlot.compute_epoch_at_slot, stateCache).len.uint64).uint64
+  if not (data.index < committee_count_at_slot):
     warn("Data index exceeds committee count",
       data_index = data.index,
-      committee_count = get_committee_count_at_slot(state, data.slot))
+      committee_count = committee_count_at_slot)
     return
 
   if not isValidAttestationTargetEpoch(state, data):

beacon_chain/spec/helpers.nim

@@ -65,7 +65,7 @@ func get_active_validator_indices*(state: BeaconState, epoch: Epoch):
       result.add idx.ValidatorIndex
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_committee_count_at_slot
-func get_committee_count_at_slot*(num_active_validators: Slot): uint64 =
+func get_committee_count_at_slot*(num_active_validators: uint64): uint64 =
   clamp(
     num_active_validators div SLOTS_PER_EPOCH div TARGET_COMMITTEE_SIZE,
     1, MAX_COMMITTEES_PER_SLOT).uint64
@@ -77,9 +77,10 @@ func get_committee_count_at_slot*(state: BeaconState, slot: Slot): uint64 =
   # be converted to CommitteeIndex types for get_beacon_committee(...); replace
   # with better and more type-safe use pattern, probably beginning with using a
   # CommitteeIndex return type here.
-  let epoch = compute_epoch_at_slot(slot)
-  let active_validator_indices = get_active_validator_indices(state, epoch)
-  result = get_committee_count_at_slot(len(active_validator_indices).uint64.Slot)
+  let
+    epoch = compute_epoch_at_slot(slot)
+    active_validator_indices = get_active_validator_indices(state, epoch)
+  result = get_committee_count_at_slot(len(active_validator_indices).uint64)
 
   # Otherwise, get_beacon_committee(...) cannot access some committees.
   doAssert (SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT).uint64 >= result

beacon_chain/spec/network.nim

@@ -70,7 +70,7 @@ func compute_subnet_for_attestation*(
   let
     slots_since_epoch_start = attestation.data.slot mod SLOTS_PER_EPOCH
     committees_since_epoch_start =
-      get_committee_count_at_slot(num_active_validators.Slot) * slots_since_epoch_start
+      get_committee_count_at_slot(num_active_validators) * slots_since_epoch_start
 
   (committees_since_epoch_start + attestation.data.index) mod ATTESTATION_SUBNET_COUNT
 

beacon_chain/spec/state_transition.nim

@@ -124,6 +124,15 @@ func process_slot*(state: var HashedBeaconState) {.nbench.} =
   state.data.block_roots[state.data.slot mod SLOTS_PER_HISTORICAL_ROOT] =
     hash_tree_root(state.data.latest_block_header)
 
+func clear_epoch_from_cache(cache: var StateCache, epoch: Epoch) =
+  cache.shuffled_active_validator_indices.del epoch
+  let
+    start_slot = epoch.compute_start_slot_at_epoch
+    end_slot = (epoch + 1).compute_start_slot_at_epoch
+
+  for i in start_slot ..< end_slot:
+    cache.beacon_proposer_indices.del i
+
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#beacon-chain-state-transition-function
 proc advance_slot*(
     state: var HashedBeaconState, updateFlags: UpdateFlags,
@@ -137,6 +146,8 @@ proc advance_slot*(
     # Note: Genesis epoch = 0, no need to test if before Genesis
     beacon_previous_validators.set(get_epoch_validator_count(state.data))
     process_epoch(state.data, updateFlags, epochCache)
+    clear_epoch_from_cache(
+      epochCache, (state.data.slot + 1).compute_epoch_at_slot)
   state.data.slot += 1
   if is_epoch_transition:
     beacon_current_validators.set(get_epoch_validator_count(state.data))
@@ -161,7 +172,7 @@ proc process_slots*(state: var HashedBeaconState, slot: Slot,
     return false
 
   # Catch up to the target slot
-  var cache = get_empty_per_epoch_cache()
+  var cache = StateCache()
   while state.data.slot < slot:
     advance_slot(state, updateFlags, cache)
 
@@ -207,10 +218,8 @@ proc state_transition*(
   #      the changes in case of failure (look out for `var BeaconState` and
   #      bool return values...)
   doAssert not rollback.isNil, "use noRollback if it's ok to mess up state"
-  when false:
-    # TODO readd this assetion when epochref cache is back
-    doAssert stateCache.shuffled_active_validator_indices.hasKey(
-      state.data.slot.compute_epoch_at_slot)
+  doAssert stateCache.shuffled_active_validator_indices.hasKey(
+    state.data.slot.compute_epoch_at_slot)
 
   if not process_slots(state, signedBlock.message.slot, flags):
     rollback(state)
@@ -255,9 +264,7 @@ proc state_transition*(
   # TODO consider moving this to testutils or similar, since non-testing
   # and fuzzing code should always be coming from blockpool which should
   # always be providing cache or equivalent
-  var cache = get_empty_per_epoch_cache()
-  # TODO not here, but in blockpool, should fill in as far ahead towards
-  # block's slot as protocol allows to be known already
+  var cache = StateCache()
   cache.shuffled_active_validator_indices[state.data.slot.compute_epoch_at_slot] =
     get_shuffled_active_validator_indices(
       state.data, state.data.slot.compute_epoch_at_slot)

beacon_chain/spec/state_transition_block.nim

@@ -274,7 +274,7 @@ proc process_voluntary_exit*(
     validator_withdrawable_epoch = validator.withdrawable_epoch,
     validator_exit_epoch = validator.exit_epoch,
     validator_effective_balance = validator.effective_balance
-  var cache = get_empty_per_epoch_cache()
+  var cache = StateCache()
   initiate_validator_exit(
     state, voluntary_exit.validator_index.ValidatorIndex, cache)
 

beacon_chain/spec/validator.nim

@@ -77,7 +77,7 @@ func get_shuffled_seq*(seed: Eth2Digest,
       if bit != 0:
         shuffled_active_validator_indices[index] = flip.ValidatorIndex
 
-  result = shuffled_active_validator_indices
+  shuffled_active_validator_indices
 
 func get_shuffled_active_validator_indices*(state: BeaconState, epoch: Epoch):
     seq[ValidatorIndex] =
@@ -90,6 +90,16 @@ func get_shuffled_active_validator_indices*(state: BeaconState, epoch: Epoch):
       active_validator_indices.len.uint64),
     active_validator_indices[it])
 
+func get_shuffled_active_validator_indices*(
+    state: BeaconState, epoch: Epoch, cache: var StateCache):
+    seq[ValidatorIndex] =
+  try:
+    cache.shuffled_active_validator_indices[epoch]
+  except KeyError:
+    let validator_indices = get_shuffled_active_validator_indices(state, epoch)
+    cache.shuffled_active_validator_indices[epoch] = validator_indices
+    validator_indices
+
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#get_previous_epoch
 func get_previous_epoch*(state: BeaconState): Epoch =
   # Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``).
@@ -142,7 +152,7 @@ func get_beacon_committee*(
 
   try:
     let committee_count = get_committee_count_at_slot(
-      cache.shuffled_active_validator_indices[epoch].len.uint64.Slot)
+      cache.shuffled_active_validator_indices[epoch].len.uint64)
     compute_committee(
       cache.shuffled_active_validator_indices[epoch],
       get_seed(state, epoch, DOMAIN_BEACON_ATTESTER),
@@ -153,10 +163,44 @@ func get_beacon_committee*(
   except KeyError:
     raiseAssert "values are added to cache before using them"
 
-# Not from spec
-func get_empty_per_epoch_cache*(): StateCache =
-  result.shuffled_active_validator_indices =
-    initTable[Epoch, seq[ValidatorIndex]]()
+# https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_shuffled_index
+func compute_shuffled_index(
+    index: uint64, index_count: uint64, seed: Eth2Digest): uint64 =
+  # Return the shuffled index corresponding to ``seed`` (and ``index_count``).
+  doAssert index < index_count
+
+  var
+    pivot_buffer: array[(32+1), byte]
+    source_buffer: array[(32+1+4), byte]
+    cur_idx_permuted = index
+
+  pivot_buffer[0..31] = seed.data
+  source_buffer[0..31] = seed.data
+
+  # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf)
+  # See the 'generalized domain' algorithm on page 3
+  for current_round in 0 ..< SHUFFLE_ROUND_COUNT.int:
+    let round_bytes1 = int_to_bytes1(current_round)[0]
+    pivot_buffer[32] = round_bytes1
+    source_buffer[32] = round_bytes1
+
+    let
+      # If using multiple indices, can amortize this
+      pivot =
+        bytes_to_int(eth2digest(pivot_buffer).data.toOpenArray(0, 7)) mod
+          index_count
+
+      flip = ((index_count + pivot) - cur_idx_permuted) mod index_count
+      position = max(cur_idx_permuted.int, flip.int)
+    source_buffer[33..36] = int_to_bytes4((position div 256).uint64)
+    let
+      source = eth2digest(source_buffer).data
+      byte_value = source[(position mod 256) div 8]
+      bit = (byte_value shr (position mod 8)) mod 2
+
+    cur_idx_permuted = if bit != 0: flip else: cur_idx_permuted
+
+  cur_idx_permuted
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/beacon-chain.md#compute_proposer_index
 func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex],
@@ -167,23 +211,19 @@ func compute_proposer_index(state: BeaconState, indices: seq[ValidatorIndex],
   if len(indices) == 0:
     return none(ValidatorIndex)
 
-  let
-    seq_len = indices.len.uint64
-    shuffled_seq = mapIt(get_shuffled_seq(seed, seq_len), indices[it])
-
-  doAssert seq_len == shuffled_seq.len.uint64
+  let seq_len = indices.len.uint64
 
   var
-    i = 0
+    i = 0'u64
     buffer: array[32+8, byte]
   buffer[0..31] = seed.data
   while true:
-    buffer[32..39] = int_to_bytes8(i.uint64 div 32)
+    buffer[32..39] = int_to_bytes8(i div 32)
     let
-      candidate_index = shuffled_seq[(i.uint64 mod seq_len).int]
+      candidate_index =
+        indices[compute_shuffled_index(i mod seq_len, seq_len, seed)]
       random_byte = (eth2digest(buffer).data)[i mod 32]
-      effective_balance =
-        state.validators[candidate_index].effective_balance
+      effective_balance = state.validators[candidate_index].effective_balance
     if effective_balance * MAX_RANDOM_BYTE >=
         MAX_EFFECTIVE_BALANCE * random_byte:
       return some(candidate_index)
@@ -245,7 +285,7 @@ func get_committee_assignment*(
   let next_epoch = get_current_epoch(state) + 1
   doAssert epoch <= next_epoch
 
-  var cache = get_empty_per_epoch_cache()
+  var cache = StateCache()
 
   let start_slot = compute_start_slot_at_epoch(epoch)
   for slot in start_slot ..< start_slot + SLOTS_PER_EPOCH:
@@ -259,7 +299,7 @@ func get_committee_assignment*(
 # https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/specs/phase0/validator.md#validator-assignments
 func is_proposer(
     state: BeaconState, validator_index: ValidatorIndex): bool {.used.} =
-  var cache = get_empty_per_epoch_cache()
+  var cache = StateCache()
   let proposer_index = get_beacon_proposer_index(state, cache)
   proposer_index.isSome and proposer_index.get == validator_index
 

beacon_chain/validator_api.nim

@@ -228,7 +228,7 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
       stateId: string, epoch: uint64, index: uint64, slot: uint64) ->
       seq[BeaconStatesCommitteesTuple]:
     withStateForStateId(stateId):
-      var cache = get_empty_per_epoch_cache() # TODO is this OK?
+      var cache = StateCache() # TODO is this OK?
 
       proc getCommittee(slot: Slot, index: CommitteeIndex): BeaconStatesCommitteesTuple =
         let vals = get_beacon_committee(state, slot, index, cache).mapIt(it.uint64)

beacon_chain/validator_duties.nim

@@ -196,7 +196,7 @@ proc makeBeaconBlockForHeadAndSlot*(node: BeaconNode,
       doAssert v.addr == addr poolPtr.tmpState.data
       assign(poolPtr.tmpState, poolPtr.headState)
 
-    var cache = get_empty_per_epoch_cache()
+    var cache = StateCache()
     let message = makeBeaconBlock(
       node.config.runtimePreset,
       hashedState,
@@ -341,13 +341,7 @@ proc handleAttestations(node: BeaconNode, head: BlockRef, slot: Slot) =
           cache.shuffled_active_validator_indices[
             slot.compute_epoch_at_slot].len.uint64
         except KeyError:
-          when false:
-            # TODO re-enable when getEpochCache() works
-            raiseAssert "getEpochCache(...) didn't fill cache"
-          let epoch = slot.compute_epoch_at_slot
-          cache.shuffled_active_validator_indices[epoch] =
-            get_shuffled_active_validator_indices(state, epoch)
-          cache.shuffled_active_validator_indices[epoch].len.uint64
+          raiseAssert "getEpochCache(...) didn't fill cache"
 
     for committee_index in 0'u64..<committees_per_slot:
       let committee = get_beacon_committee(
@@ -407,7 +401,7 @@ proc broadcastAggregatedAttestations(
   node.blockPool.withState(node.blockPool.tmpState, bs):
     let
       committees_per_slot = get_committee_count_at_slot(state, aggregationSlot)
-    var cache = get_empty_per_epoch_cache()
+    var cache = StateCache()
     for committee_index in 0'u64..<committees_per_slot:
       let committee = get_beacon_committee(
         state, aggregationSlot, committee_index.CommitteeIndex, cache)