Shard fork choice rule

setup.py

@@ -378,6 +378,7 @@ def finalize_options(self):
                     specs/phase1/shard-transition.md
                     specs/phase1/fork-choice.md
                     specs/phase1/phase1-fork.md
+                    specs/phase1/shard-fork-choice.md
                 """
             else:
                 raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork)

specs/phase1/fork-choice.md

@@ -10,6 +10,9 @@
 
 - [Introduction](#introduction)
 - [Fork choice](#fork-choice)
+  - [Helpers](#helpers)
+    - [Extended `LatestMessage`](#extended-latestmessage)
+    - [Updated `update_latest_messages`](#updated-update_latest_messages)
   - [Handlers](#handlers)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -25,6 +28,33 @@ Due to the changes in the structure of `IndexedAttestation` in Phase 1, `on_atte
 
 The rest of the fork choice remains stable.
 
+### Helpers
+
+#### Extended `LatestMessage`
+
+```python
+@dataclass(eq=True, frozen=True)
+class LatestMessage(object):
+    epoch: Epoch
+    root: Root
+    shard: Shard
+    shard_root: Root
+```
+
+#### Updated `update_latest_messages`
+
+```python
+def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None:
+    target = attestation.data.target
+    beacon_block_root = attestation.data.beacon_block_root
+    shard = get_shard(store.block_states[beacon_block_root], attestation)
+    for i in attesting_indices:
+        if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch:
+            store.latest_messages[i] = LatestMessage(
+                epoch=target.epoch, root=beacon_block_root, shard=shard, shard_root=attestation.data.head_shard_root
+            )
+```
+
 ### Handlers
 
 ```python
@@ -49,4 +79,4 @@ def on_attestation(store: Store, attestation: Attestation) -> None:
         if attestation.aggregation_bits[i]
     ]
     update_latest_messages(store, attesting_indices, attestation)
-```
+```

specs/phase1/shard-fork-choice.md

@@ -0,0 +1,138 @@
+# Ethereum 2.0 Phase 1 -- Beacon Chain + Shard Chain Fork Choice
+
+**Notice**: This document is a work-in-progress for researchers and implementers.
+
+## Table of contents
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+
+- [Introduction](#introduction)
+- [Fork choice](#fork-choice)
+  - [Helpers](#helpers)
+    - [`ShardStore`](#shardstore)
+    - [`get_forkchoice_shard_store`](#get_forkchoice_shard_store)
+    - [`get_shard_latest_attesting_balance`](#get_shard_latest_attesting_balance)
+    - [`get_shard_head`](#get_shard_head)
+    - [`get_shard_ancestor`](#get_shard_ancestor)
+  - [Handlers](#handlers)
+    - [`on_shard_block`](#on_shard_block)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Introduction
+
+This document is the shard chain fork choice spec for part of Ethereum 2.0 Phase 1.
+
+## Fork choice
+
+### Helpers
+
+#### `ShardStore`
+
+```python
+@dataclass
+class ShardStore:
+    shard: Shard
+    blocks: Dict[Root, ShardBlock] = field(default_factory=dict)
+    block_states: Dict[Root, ShardState] = field(default_factory=dict)
+```
+
+#### `get_forkchoice_shard_store`
+
+```python
+def get_forkchoice_shard_store(anchor_state: BeaconState, shard: Shard) -> ShardStore:
+    return ShardStore(
+        shard=shard,
+        blocks={anchor_state.shard_states[shard].latest_block_root: ShardBlock(slot=anchor_state.slot)},
+        block_states={anchor_state.shard_states[shard].latest_block_root: anchor_state.copy().shard_states[shard]},
+    )
+```
+
+#### `get_shard_latest_attesting_balance`
+
+```python
+def get_shard_latest_attesting_balance(store: Store, shard_store: ShardStore, root: Root) -> Gwei:
+    state = store.checkpoint_states[store.justified_checkpoint]
+    active_indices = get_active_validator_indices(state, get_current_epoch(state))
+    return Gwei(sum(
+        state.validators[i].effective_balance for i in active_indices
+        if (
+            i in store.latest_messages and get_shard_ancestor(
+                store, shard_store, store.latest_messages[i].root, shard_store.blocks[root].slot
+            ) == root
+        )
+    ))
+```
+
+#### `get_shard_head`
+
+```python
+def get_shard_head(store: Store, shard_store: ShardStore) -> Root:
+    # Execute the LMD-GHOST fork choice
+    head_beacon_root = get_head(store)
+    head_shard_root = store.block_states[head_beacon_root].shard_states[shard_store.shard].latest_block_root
+    while True:
+        children = [
+            root for root in shard_store.blocks.keys()
+            if shard_store.blocks[root].shard_parent_root == head_shard_root
+        ]
+        if len(children) == 0:
+            return head_shard_root
+        # Sort by latest attesting balance with ties broken lexicographically
+        head_shard_root = max(
+            children, key=lambda root: (get_shard_latest_attesting_balance(store, shard_store, root), root)
+        )
+```
+
+#### `get_shard_ancestor`
+
+```python
+def get_shard_ancestor(store: Store, shard_store: ShardStore, root: Root, slot: Slot) -> Root:
+    block = shard_store.blocks[root]
+    if block.slot > slot:
+        return get_shard_ancestor(store, shard_store, block.shard_parent_root, slot)
+    elif block.slot == slot:
+        return root
+    else:
+        # root is older than queried slot, thus a skip slot. Return earliest root prior to slot
+        return root
+```
+
+### Handlers
+
+#### `on_shard_block`
+
+```python
+def on_shard_block(store: Store, shard_store: ShardStore, signed_shard_block: SignedShardBlock) -> None:
+    shard_block = signed_shard_block.message
+    shard = shard_store.shard
+    # 1. Check shard parent exists
+    assert shard_block.shard_parent_root in shard_store.block_states
+    pre_shard_state = shard_store.block_states[shard_block.shard_parent_root]
+
+    # 2. Check beacon parent exists
+    assert shard_block.beacon_parent_root in store.block_states
+    beacon_state = store.block_states[shard_block.beacon_parent_root]
+
+    # 3. Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor)
+    finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+    assert shard_block.slot > finalized_slot
+
+    # 4. Check block is a descendant of the finalized block at the checkpoint finalized slot
+    assert (
+        shard_block.beacon_parent_root == store.finalized_checkpoint.root
+        or get_ancestor(store, shard_block.beacon_parent_root, finalized_slot) == store.finalized_checkpoint.root
+    )
+
+    # Add new block to the store
+    shard_store.blocks[hash_tree_root(shard_block)] = shard_block
+
+    # Check the block is valid and compute the post-state
+    verify_shard_block_message(beacon_state, pre_shard_state, shard_block, shard_block.slot, shard)
+    verify_shard_block_signature(beacon_state, signed_shard_block)
+    post_state = get_post_shard_state(beacon_state, pre_shard_state, shard_block)
+    # Add new state for this block to the store
+    shard_store.block_states[hash_tree_root(shard_block)] = post_state
+```

tests/core/pyspec/eth2spec/test/fork_choice/test_get_head.py

@@ -1,41 +1,13 @@
 from eth2spec.test.context import with_all_phases, spec_state_test
 from eth2spec.test.helpers.attestations import get_valid_attestation, next_epoch_with_attestations
 from eth2spec.test.helpers.block import build_empty_block_for_next_slot
+from eth2spec.test.helpers.fork_choice import add_attestation_to_store, add_block_to_store, get_anchor_root
 from eth2spec.test.helpers.state import (
     next_epoch,
     state_transition_and_sign_block,
 )
 
 
-def add_block_to_store(spec, store, signed_block):
-    pre_state = store.block_states[signed_block.message.parent_root]
-    block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT
-
-    if store.time < block_time:
-        spec.on_tick(store, block_time)
-
-    spec.on_block(store, signed_block)
-
-
-def add_attestation_to_store(spec, store, attestation):
-    parent_block = store.blocks[attestation.data.beacon_block_root]
-    pre_state = store.block_states[spec.hash_tree_root(parent_block)]
-    block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT
-    next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT
-
-    if store.time < next_epoch_time:
-        spec.on_tick(store, next_epoch_time)
-
-    spec.on_attestation(store, attestation)
-
-
-def get_anchor_root(spec, state):
-    anchor_block_header = state.latest_block_header.copy()
-    if anchor_block_header.state_root == spec.Bytes32():
-        anchor_block_header.state_root = spec.hash_tree_root(state)
-    return spec.hash_tree_root(anchor_block_header)
-
-
 @with_all_phases
 @spec_state_test
 def test_genesis(spec, state):

tests/core/pyspec/eth2spec/test/fork_choice/test_on_attestation.py

@@ -18,18 +18,25 @@ def run_on_attestation(spec, state, store, attestation, valid=True):
 
     if spec.fork == PHASE0:
         sample_index = indexed_attestation.attesting_indices[0]
+        latest_message = spec.LatestMessage(
+            epoch=attestation.data.target.epoch,
+            root=attestation.data.beacon_block_root,
+        )
     else:
         attesting_indices = [
             index for i, index in enumerate(indexed_attestation.committee)
             if attestation.aggregation_bits[i]
         ]
         sample_index = attesting_indices[0]
-    assert (
-        store.latest_messages[sample_index] ==
-        spec.LatestMessage(
+        latest_message = spec.LatestMessage(
             epoch=attestation.data.target.epoch,
             root=attestation.data.beacon_block_root,
+            shard=spec.get_shard(state, attestation),
+            shard_root=attestation.data.head_shard_root,
         )
+
+    assert (
+        store.latest_messages[sample_index] == latest_message
     )
 
 

tests/core/pyspec/eth2spec/test/fork_choice/test_on_shard_head.py

@@ -0,0 +1,85 @@
+from eth2spec.utils.ssz.ssz_impl import hash_tree_root
+
+from eth2spec.test.context import spec_state_test, with_all_phases_except, PHASE0
+from eth2spec.test.helpers.shard_block import (
+    build_attestation_with_shard_transition,
+    build_shard_block,
+    build_shard_transitions_till_slot,
+)
+from eth2spec.test.helpers.fork_choice import add_block_to_store, get_anchor_root
+from eth2spec.test.helpers.state import next_slot, state_transition_and_sign_block
+from eth2spec.test.helpers.block import build_empty_block
+
+
+def run_on_shard_block(spec, store, shard_store, signed_block, valid=True):
+    if not valid:
+        try:
+            spec.on_shard_block(store, shard_store, signed_block)
+        except AssertionError:
+            return
+        else:
+            assert False
+
+    spec.on_shard_block(store, shard_store, signed_block)
+    assert shard_store.blocks[hash_tree_root(signed_block.message)] == signed_block.message
+
+
+def run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index):
+    shard = shard_store.shard
+    store.time = store.time + spec.SECONDS_PER_SLOT * spec.SLOTS_PER_EPOCH
+
+    # Create SignedShardBlock
+    body = b'\x56' * spec.MAX_SHARD_BLOCK_SIZE
+    shard_block = build_shard_block(spec, state, shard, body=body, signed=True)
+    shard_blocks = [shard_block]
+
+    # Attester creates `attestation`
+    # Use temporary next state to get ShardTransition of shard block
+    shard_transitions = build_shard_transitions_till_slot(
+        spec,
+        state,
+        shards=[shard, ],
+        shard_blocks={shard: shard_blocks},
+        target_len_offset_slot=1,
+    )
+    shard_transition = shard_transitions[shard]
+    attestation = build_attestation_with_shard_transition(
+        spec,
+        state,
+        slot=state.slot,
+        index=committee_index,
+        target_len_offset_slot=1,
+        shard_transition=shard_transition,
+    )
+
+    # Propose beacon block at slot
+    beacon_block = build_empty_block(spec, state, slot=state.slot + 1)
+    beacon_block.body.attestations = [attestation]
+    beacon_block.body.shard_transitions = shard_transitions
+    signed_beacon_block = state_transition_and_sign_block(spec, state, beacon_block)
+
+    run_on_shard_block(spec, store, shard_store, shard_block)
+    add_block_to_store(spec, store, signed_beacon_block)
+
+    assert spec.get_head(store) == beacon_block.hash_tree_root()
+    assert spec.get_shard_head(store, shard_store) == shard_block.message.hash_tree_root()
+
+
+@with_all_phases_except([PHASE0])
+@spec_state_test
+def test_basic(spec, state):
+    spec.PHASE_1_GENESIS_SLOT = 0  # FIXME: remove mocking
+    state = spec.upgrade_to_phase1(state)
+    next_slot(spec, state)
+
+    # Initialization
+    store = spec.get_forkchoice_store(state)
+    anchor_root = get_anchor_root(spec, state)
+    assert spec.get_head(store) == anchor_root
+
+    committee_index = spec.CommitteeIndex(0)
+    shard = spec.compute_shard_from_committee_index(state, committee_index, state.slot)
+    shard_store = spec.get_forkchoice_shard_store(state, shard)
+
+    run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index)
+    run_apply_shard_and_beacon(spec, state, store, shard_store, committee_index)

tests/core/pyspec/eth2spec/test/helpers/fork_choice.py

@@ -0,0 +1,27 @@
+def get_anchor_root(spec, state):
+    anchor_block_header = state.latest_block_header.copy()
+    if anchor_block_header.state_root == spec.Bytes32():
+        anchor_block_header.state_root = spec.hash_tree_root(state)
+    return spec.hash_tree_root(anchor_block_header)
+
+
+def add_block_to_store(spec, store, signed_block):
+    pre_state = store.block_states[signed_block.message.parent_root]
+    block_time = pre_state.genesis_time + signed_block.message.slot * spec.SECONDS_PER_SLOT
+
+    if store.time < block_time:
+        spec.on_tick(store, block_time)
+
+    spec.on_block(store, signed_block)
+
+
+def add_attestation_to_store(spec, store, attestation):
+    parent_block = store.blocks[attestation.data.beacon_block_root]
+    pre_state = store.block_states[spec.hash_tree_root(parent_block)]
+    block_time = pre_state.genesis_time + parent_block.slot * spec.SECONDS_PER_SLOT
+    next_epoch_time = block_time + spec.SLOTS_PER_EPOCH * spec.SECONDS_PER_SLOT
+
+    if store.time < next_epoch_time:
+        spec.on_tick(store, next_epoch_time)
+
+    spec.on_attestation(store, attestation)