udpate validator guide to work with all updated phase 1 constructions

specs/phase1/beacon-chain.md

@@ -854,6 +854,9 @@ def apply_shard_transition(state: BeaconState, shard: Shard, transition: ShardTr
             shard_parent_root = hash_tree_root(header)
             headers.append(header)
             proposers.append(proposal_index)
+        else:
+            # Must have a stub for `shard_data_root` if empty slot
+            assert transition.shard_data_roots[i] == Root()
 
         prev_gasprice = shard_state.gasprice
 

specs/phase1/shard-transition.md

@@ -154,141 +154,3 @@ def generate_custody_bit(subkey: BLSPubkey, block: ShardBlock) -> bool:
     # TODO
     ...
 ```
-
-## Honest committee member behavior
-
-### Helper functions
-
-```python
-def get_winning_proposal(beacon_state: BeaconState, proposals: Sequence[SignedShardBlock]) -> SignedShardBlock:
-    # TODO: Let `winning_proposal` be the proposal with the largest number of total attestations from slots in
-    # `state.shard_next_slots[shard]....slot-1` supporting it or any of its descendants, breaking ties by choosing
-    # the first proposal locally seen. Do `proposals.append(winning_proposal)`.
-    return proposals[-1]  # stub
-```
-
-```python
-def compute_shard_body_roots(proposals: Sequence[SignedShardBlock]) -> Sequence[Root]:
-    return [hash_tree_root(proposal.message.body) for proposal in proposals]
-```
-
-```python
-def get_proposal_choices_at_slot(beacon_state: BeaconState,
-                                 shard_state: ShardState,
-                                 slot: Slot,
-                                 shard: Shard,
-                                 shard_blocks: Sequence[SignedShardBlock],
-                                 validate_signature: bool=True) -> Sequence[SignedShardBlock]:
-    """
-    Return the valid shard blocks at the given ``slot``.
-    Note that this function doesn't change the state.
-    """
-    choices = []
-    shard_blocks_at_slot = [block for block in shard_blocks if block.message.slot == slot]
-    for block in shard_blocks_at_slot:
-        try:
-            # Verify block message and signature
-            # TODO these validations should have been checked upon receiving shard blocks.
-            assert verify_shard_block_message(beacon_state, shard_state, block.message, slot, shard)
-            if validate_signature:
-                assert verify_shard_block_signature(beacon_state, block)
-
-            shard_state = get_post_shard_state(beacon_state, shard_state, block.message)
-        except Exception:
-            pass  # TODO: throw error in the test helper
-        else:
-            choices.append(block)
-    return choices
-```
-
-```python
-def get_proposal_at_slot(beacon_state: BeaconState,
-                         shard_state: ShardState,
-                         slot: Shard,
-                         shard: Shard,
-                         shard_blocks: Sequence[SignedShardBlock],
-                         validate_signature: bool=True) -> Tuple[SignedShardBlock, ShardState]:
-    """
-    Return ``proposal``, ``shard_state`` of the given ``slot``.
-    Note that this function doesn't change the state.
-    """
-    choices = get_proposal_choices_at_slot(
-        beacon_state=beacon_state,
-        shard_state=shard_state,
-        slot=slot,
-        shard=shard,
-        shard_blocks=shard_blocks,
-        validate_signature=validate_signature,
-    )
-    if len(choices) == 0:
-        block = ShardBlock(slot=slot)
-        proposal = SignedShardBlock(message=block)
-    elif len(choices) == 1:
-        proposal = choices[0]
-    else:
-        proposal = get_winning_proposal(beacon_state, choices)
-
-    # Apply state transition
-    shard_state = get_post_shard_state(beacon_state, shard_state, proposal.message)
-
-    return proposal, shard_state
-```
-
-```python
-def get_shard_state_transition_result(
-    beacon_state: BeaconState,
-    shard: Shard,
-    shard_blocks: Sequence[SignedShardBlock],
-    validate_signature: bool=True,
-) -> Tuple[Sequence[SignedShardBlock], Sequence[ShardState], Sequence[Root]]:
-    proposals = []
-    shard_states = []
-    shard_state = beacon_state.shard_states[shard]
-    for slot in get_offset_slots(beacon_state, shard):
-        proposal, shard_state = get_proposal_at_slot(
-            beacon_state=beacon_state,
-            shard_state=shard_state,
-            slot=slot,
-            shard=shard,
-            shard_blocks=shard_blocks,
-            validate_signature=validate_signature,
-        )
-        shard_states.append(shard_state)
-        proposals.append(proposal)
-
-    shard_data_roots = compute_shard_body_roots(proposals)
-
-    return proposals, shard_states, shard_data_roots
-```
-
-### Make attestations
-
-Suppose you are a committee member on shard `shard` at slot `current_slot` and you have received shard blocks `shard_blocks` since the latest successful crosslink for `shard` into the beacon chain. Let `beacon_state` be the head beacon state you are building on, and let `QUARTER_PERIOD = SECONDS_PER_SLOT // 4`. `2 * QUARTER_PERIOD` seconds into slot `current_slot`, run `get_shard_transition(beacon_state, shard, shard_blocks)` to get `shard_transition`.
-
-```python
-def get_shard_transition(beacon_state: BeaconState,
-                         shard: Shard,
-                         shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition:
-    offset_slots = get_offset_slots(beacon_state, shard)
-    proposals, shard_states, shard_data_roots = get_shard_state_transition_result(beacon_state, shard, shard_blocks)
-
-    shard_block_lengths = []
-    proposer_signatures = []
-    for proposal in proposals:
-        shard_block_lengths.append(len(proposal.message.body))
-        if proposal.signature != NO_SIGNATURE:
-            proposer_signatures.append(proposal.signature)
-
-    if len(proposer_signatures) > 0:
-        proposer_signature_aggregate = bls.Aggregate(proposer_signatures)
-    else:
-        proposer_signature_aggregate = NO_SIGNATURE
-
-    return ShardTransition(
-        start_slot=offset_slots[0],
-        shard_block_lengths=shard_block_lengths,
-        shard_data_roots=shard_data_roots,
-        shard_states=shard_states,
-        proposer_signature_aggregate=proposer_signature_aggregate,
-    )
-```

specs/phase1/validator.md

@@ -189,7 +189,7 @@ def get_best_light_client_aggregate(block: BeaconBlock,
                                     aggregates: Sequence[LightClientVote]) -> LightClientVote:
     viable_aggregates = [
         aggregate for aggregate in aggregates
-        if aggregate.slot == get_previous_slot(block.slot) and aggregate.beacon_block_root == block.parent_root
+        if aggregate.slot == compute_previous_slot(block.slot) and aggregate.beacon_block_root == block.parent_root
     ]
 
     return max(
@@ -242,7 +242,7 @@ class FullAttestation(Container):
 
 Note the timing of when to create/broadcast is altered from Phase 1.
 
-A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid `BeaconBlock` from the expected beacon block proposer and a valid `ShardBlock` for the expected shard block porposer for the assigned `slot` or (b) one-half of the `slot` has transpired (`SECONDS_PER_SLOT / 2` seconds after the start of `slot`) -- whichever comes _first_.
+A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid `BeaconBlock` from the expected beacon block proposer and a valid `ShardBlock` for the expected shard block proposer for the assigned `slot` or (b) one-half of the `slot` has transpired (`SECONDS_PER_SLOT / 2` seconds after the start of `slot`) -- whichever comes _first_.
 
 #### Attestation data
 
@@ -251,24 +251,67 @@ A validator should create and broadcast the `attestation` to the associated atte
 - Let `head_block` be the result of running the fork choice during the assigned slot.
 - Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`.
 - Let `head_shard_block` be the result of running the fork choice on the assigned shard chain during the assigned slot.
+- Let `shard_blocks` be the shard blocks in the chain starting immediately _after_ the most recent crosslink (`head_state.shard_transitions[shard].latest_block_root`) up to the `head_shard_block`.
+
+*Note*: We assume that the fork choice only follows branches with valid `offset_slots` with respect to the most recent beacon state shard transition for the queried shard.
 
 ##### Head shard root
 
 Set `attestation_data.head_shard_root = hash_tree_root(head_shard_block)`.
 
 ##### Shard transition
 
-Set `shard_transition` to the value returned by `get_shard_transition()`.
+Set `shard_transition` to the value returned by `get_shard_transition(head_state, shard, shard_blocks)`.
 
 ```python
-def get_shard_transition(state: BeaconState,
+def get_shard_state_transition_result(
+    beacon_state: BeaconState,
+    shard: Shard,
+    shard_blocks: Sequence[SignedShardBlock],
+    validate_signature: bool=True,
+) -> Tuple[Sequence[ShardState], Sequence[Root], Sequence[uint64]]:
+    shard_states = []
+    shard_data_roots = []
+    shard_block_lengths = []
+
+    shard_state = beacon_state.shard_states[shard]
+    shard_block_slots = [shard_block.message.slot for shard_block in shard_blocks]
+    for slot in get_offset_slots(beacon_state, shard):
+        if slot in shard_block_slots:
+            shard_block = shard_blocks[shard_block_slots.index(slot)]
+            shard_data_roots.append(hash_tree_root(shard_block.message.body))
+        else:
+            shard_block = SignedShardBlock(message=ShardBlock(slot=slot))
+            shard_data_roots.append(Root())
+        shard_state = get_post_shard_state(beacon_state, shard_state, shard_block.message)
+        shard_states.append(shard_state)
+        shard_block_lengths.append(len(shard_block.message.body))
+
+    return shard_states, shard_data_roots, shard_block_lengths
+```
+
+```python
+def get_shard_transition(beacon_state: BeaconState,
                          shard: Shard,
-                         shard_blocks: Sequence[ShardBlockWrapper]) -> ShardTransition:
-    """
-    latest_shard_slot = get_latest_slot_for_shard(state, shard)
-    offset_slots = [Slot(latest_shard_slot + x) for x in SHARD_BLOCK_OFFSETS if latest_shard_slot + x <= state.slot]
-    """
-    return ShardTransition()
+                         shard_blocks: Sequence[SignedShardBlock]) -> ShardTransition:
+    offset_slots = get_offset_slots(beacon_state, shard)
+    shard_states, shard_data_roots, shard_block_lengths = (
+        get_shard_state_transition_result(beacon_state, shard, shard_blocks)
+    )
+
+    if len(shard_blocks) > 0:
+        proposer_signatures = [shard_block.signature for shard_block in shard_blocks]
+        proposer_signature_aggregate = bls.Aggregate(proposer_signatures)
+    else:
+        proposer_signature_aggregate = NO_SIGNATURE
+
+    return ShardTransition(
+        start_slot=offset_slots[0],
+        shard_block_lengths=shard_block_lengths,
+        shard_data_roots=shard_data_roots,
+        shard_states=shard_states,
+        proposer_signature_aggregate=proposer_signature_aggregate,
+    )
 ```
 
 #### Construct attestation
@@ -292,10 +335,25 @@ Set `attestation.signature = attestation_signature` where `attestation_signature
 
 ```python
 def get_attestation_signature(state: BeaconState,
-                              attestation_data: AttestationData,
-                              cb_blocks: List[Bitlist[MAX_VALIDATORS_PER_COMMITTEE], MAX_SHARD_BLOCKS_PER_ATTESTATION],
+                              attestation: Attestation,
                               privkey: int) -> BLSSignature:
-    pass
+    domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch)
+    attestation_data_root = hash_tree_root(attestation.data)
+    index_in_committee = attestation.aggregation_bits.index(True)
+    signatures = []
+    for block_index, custody_bits in enumerate(attestation.custody_bits_blocks):
+        custody_bit = custody_bits[index_in_committee]
+        signing_root = compute_signing_root(
+            AttestationCustodyBitWrapper(
+                attestation_data_root=attestation_data_root,
+                block_index=block_index,
+                bit=custody_bit,
+            ),
+            domain,
+        )
+        signatures.append(bls.Sign(privkey, signing_root))
+
+    return bls.Aggregate(signatures)
 ```
 
 ### Light client committee

tests/core/pyspec/eth2spec/test/validator/test_validator_unittest.py

@@ -1,5 +1,11 @@
-from eth2spec.test.context import spec_state_test, always_bls, with_all_phases
-from eth2spec.test.helpers.attestations import build_attestation_data
+from random import Random
+
+from eth2spec.test.context import (
+    spec_state_test,
+    always_bls, with_phases, with_all_phases, with_all_phases_except,
+    PHASE0,
+)
+from eth2spec.test.helpers.attestations import build_attestation_data, get_valid_attestation
 from eth2spec.test.helpers.block import build_empty_block
 from eth2spec.test.helpers.deposits import prepare_state_and_deposit
 from eth2spec.test.helpers.keys import privkeys, pubkeys
@@ -317,25 +323,48 @@ def test_get_block_signature(spec, state):
 # Attesting
 
 
-@with_all_phases
+@with_phases([PHASE0])
 @spec_state_test
 @always_bls
-def test_get_attestation_signature(spec, state):
+def test_get_attestation_signature_phase0(spec, state):
     privkey = privkeys[0]
     pubkey = pubkeys[0]
-    attestation_data = spec.AttestationData(slot=10)
-    domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch)
+    attestation = get_valid_attestation(spec, state, signed=False)
+    domain = spec.get_domain(state, spec.DOMAIN_BEACON_ATTESTER, attestation.data.target.epoch)
+
     run_get_signature_test(
         spec=spec,
         state=state,
-        obj=attestation_data,
+        obj=attestation.data,
         domain=domain,
         get_signature_fn=spec.get_attestation_signature,
         privkey=privkey,
         pubkey=pubkey,
     )
 
 
+@with_all_phases_except([PHASE0])
+@spec_state_test
+@always_bls
+def test_get_attestation_signature_phase1plus(spec, state):
+    privkey = privkeys[0]
+
+    def single_participant(comm):
+        rng = Random(1100)
+        return rng.sample(comm, 1)
+
+    attestation = get_valid_attestation(spec, state, filter_participant_set=single_participant, signed=False)
+    indexed_attestation = spec.get_indexed_attestation(state, attestation)
+
+    assert indexed_attestation.attestation.aggregation_bits.count(True) == 1
+
+    # Cannot use normal `run_get_signature_test` due to complex signature type
+    index_in_committee = indexed_attestation.attestation.aggregation_bits.index(True)
+    privkey = privkeys[indexed_attestation.committee[index_in_committee]]
+    attestation.signature = spec.get_attestation_signature(state, attestation, privkey)
+    assert spec.verify_attestation_custody(state, spec.get_indexed_attestation(state, attestation))
+
+
 # Attestation aggregation
 
 
@@ -363,7 +392,7 @@ def test_get_slot_signature(spec, state):
 @always_bls
 def test_is_aggregator(spec, state):
     # TODO: we can test the probabilistic result against `TARGET_AGGREGATORS_PER_COMMITTEE`
-    #  if we have more validators and larger committeee size
+    #  if we have more validators and larger committee size
     slot = state.slot
     committee_index = 0
     has_aggregator = False
@@ -377,7 +406,7 @@ def test_is_aggregator(spec, state):
     assert has_aggregator
 
 
-@with_all_phases
+@with_phases([PHASE0])
 @spec_state_test
 @always_bls
 def test_get_aggregate_signature(spec, state):