Merge branch 'dev' into vbuterin-patch-12

ethereum · Feb 10, 2019 · 1e1e3ad · 1e1e3ad
2 parents c7891c8 + 2944a7d
commit 1e1e3ad
Show file tree

Hide file tree

Showing 3 changed files with 106 additions and 105 deletions.
diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md
@@ -60,7 +60,7 @@
         - [`get_epoch_start_slot`](#get_epoch_start_slot)
         - [`is_active_validator`](#is_active_validator)
         - [`get_active_validator_indices`](#get_active_validator_indices)
-        - [`shuffle`](#shuffle)
+        - [`get_permuted_index`](#get_permuted_index)
         - [`split`](#split)
         - [`get_epoch_committee_count`](#get_epoch_committee_count)
         - [`get_shuffling`](#get_shuffling)
@@ -77,6 +77,7 @@
         - [`get_attestation_participants`](#get_attestation_participants)
         - [`is_power_of_two`](#is_power_of_two)
         - [`int_to_bytes1`, `int_to_bytes2`, ...](#int_to_bytes1-int_to_bytes2-)
+        - [`bytes_to_int`](#bytes_to_int)
         - [`get_effective_balance`](#get_effective_balance)
         - [`get_total_balance`](#get_total_balance)
         - [`get_fork_version`](#get_fork_version)
@@ -124,7 +125,7 @@
                 - [Deposits](#deposits-1)
                 - [Exits](#exits-1)
         - [Per-epoch processing](#per-epoch-processing)
-            - [Helpers](#helpers)
+            - [Helper variables](#helper-variables)
             - [Eth1 data](#eth1-data-1)
             - [Justification](#justification)
             - [Crosslinks](#crosslinks)
@@ -184,6 +185,7 @@ Code snippets appearing in `this style` are to be interpreted as Python code. Be
 | `BEACON_CHAIN_SHARD_NUMBER` | `2**64 - 1` | - |
 | `MAX_INDICES_PER_SLASHABLE_VOTE` | `2**12` (= 4,096) | votes |
 | `MAX_WITHDRAWALS_PER_EPOCH` | `2**2` (= 4) | withdrawals |
+| `SHUFFLE_ROUND_COUNT` | 90 | - |
 
 * For the safety of crosslinks `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `EPOCH_LENGTH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.)
 
@@ -362,8 +364,8 @@ The following data structures are defined as [SimpleSerialize (SSZ)](https://git
     'epoch_boundary_root': 'bytes32',
     # Shard block's hash of root
     'shard_block_root': 'bytes32',
-    # Last crosslink's hash of root
-    'latest_crosslink_root': 'bytes32',
+    # Last crosslink
+    'latest_crosslink': Crosslink,
     # Last justified epoch in the beacon state
     'justified_epoch': 'uint64',
     # Hash of the last justified beacon block
@@ -524,6 +526,7 @@ The following data structures are defined as [SimpleSerialize (SSZ)](https://git
     # Ethereum 1.0 chain data
     'latest_eth1_data': Eth1Data,
     'eth1_data_votes': [Eth1DataVote],
+    'deposit_index': 'uint64'
 }
 ```
 
@@ -656,9 +659,10 @@ def get_previous_epoch(state: BeaconState) -> EpochNumber:
     Return the previous epoch of the given ``state``.
     If the current epoch is  ``GENESIS_EPOCH``, return ``GENESIS_EPOCH``.
     """
-    if slot_to_epoch(state.slot) > GENESIS_EPOCH:
-        return slot_to_epoch(state.slot) - 1
-    return slot_to_epoch(state.slot)
+    current_epoch = get_current_epoch(state)
+    if current_epoch == GENESIS_EPOCH:
+        return GENESIS_EPOCH
+    return current_epoch - 1
 ```
 
 ### `get_current_epoch`
@@ -700,57 +704,27 @@ def get_active_validator_indices(validators: List[Validator], epoch: EpochNumber
     return [i for i, v in enumerate(validators) if is_active_validator(v, epoch)]
 ```
 
-### `shuffle`
+### `get_permuted_index`
 
 ```python
-def shuffle(values: List[Any], seed: Bytes32) -> List[Any]:
-    """
-    Return the shuffled ``values`` with ``seed`` as entropy.
+def get_permuted_index(index: int, list_size: int, seed: Bytes32) -> int:
     """
-    values_count = len(values)
-
-    # Entropy is consumed from the seed in 3-byte (24 bit) chunks.
-    rand_bytes = 3
-    # The highest possible result of the RNG.
-    rand_max = 2 ** (rand_bytes * 8) - 1
-
-    # The range of the RNG places an upper-bound on the size of the list that
-    # may be shuffled. It is a logic error to supply an oversized list.
-    assert values_count < rand_max
-
-    output = [x for x in values]
-    source = seed
-    index = 0
-    while index < values_count - 1:
-        # Re-hash the `source` to obtain a new pattern of bytes.
-        source = hash(source)
-        # Iterate through the `source` bytes in 3-byte chunks.
-        for position in range(0, 32 - (32 % rand_bytes), rand_bytes):
-            # Determine the number of indices remaining in `values` and exit
-            # once the last index is reached.
-            remaining = values_count - index
-            if remaining == 1:
-                break
-
-            # Read 3-bytes of `source` as a 24-bit big-endian integer.
-            sample_from_source = int.from_bytes(source[position:position + rand_bytes], 'big')
-
-            # Sample values greater than or equal to `sample_max` will cause
-            # modulo bias when mapped into the `remaining` range.
-            sample_max = rand_max - rand_max % remaining
+    Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy.
 
-            # Perform a swap if the consumed entropy will not cause modulo bias.
-            if sample_from_source < sample_max:
-                # Select a replacement index for the current index.
-                replacement_position = (sample_from_source % remaining) + index
-                # Swap the current index with the replacement index.
-                output[index], output[replacement_position] = output[replacement_position], output[index]
-                index += 1
-            else:
-                # The sample causes modulo bias. A new sample should be read.
-                pass
+    Utilizes 'swap or not' shuffling found in
+    https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
+    See the 'generalized domain' algorithm on page 3.
+    """
+    for round in range(SHUFFLE_ROUND_COUNT):
+        pivot = bytes_to_int(hash(seed + int_to_bytes1(round))[0:8]) % list_size
+        flip = (pivot - index) % list_size
+        position = max(index, flip)
+        source = hash(seed + int_to_bytes1(round) + int_to_bytes4(position // 256))
+        byte = source[(position % 256) // 8]
+        bit = (byte >> (position % 8)) % 2
+        index = flip if bit else index
 
-    return output
+    return index
 ```
 
 ### `split`
@@ -800,7 +774,10 @@ def get_shuffling(seed: Bytes32,
     committees_per_epoch = get_epoch_committee_count(len(active_validator_indices))
 
     # Shuffle
-    shuffled_active_validator_indices = shuffle(active_validator_indices, seed)
+    shuffled_active_validator_indices = [
+        active_validator_indices[get_permuted_index(i, len(active_validator_indices), seed)]
+        for i in active_validator_indices
+    ]
 
     # Split the shuffled list into committees_per_epoch pieces
     return split(shuffled_active_validator_indices, committees_per_epoch)
@@ -1037,7 +1014,14 @@ def is_power_of_two(value: int) -> bool:
 
 ### `int_to_bytes1`, `int_to_bytes2`, ...
 
-`int_to_bytes1(x): return x.to_bytes(1, 'big')`, `int_to_bytes2(x): return x.to_bytes(2, 'big')`, and so on for all integers, particularly 1, 2, 3, 4, 8, 32, 48, 96.
+`int_to_bytes1(x): return x.to_bytes(1, 'little')`, `int_to_bytes2(x): return x.to_bytes(2, 'little')`, and so on for all integers, particularly 1, 2, 3, 4, 8, 32, 48, 96.
+
+### `bytes_to_int`
+
+```python
+def bytes_to_int(data: bytes) -> int:
+    return int.from_bytes(data, 'little')
+```
 
 ### `get_effective_balance`
 
@@ -1270,13 +1254,16 @@ def process_deposit(state: BeaconState,
     Note that this function mutates ``state``.
     """
     # Validate the given `proof_of_possession`
-    assert validate_proof_of_possession(
+    proof_is_valid = validate_proof_of_possession(
         state,
         pubkey,
         proof_of_possession,
         withdrawal_credentials,
     )
 
+    if not proof_is_valid:
+        return
+
     validator_pubkeys = [v.pubkey for v in state.validator_registry]
 
     if pubkey not in validator_pubkeys:
@@ -1495,6 +1482,7 @@ def get_initial_beacon_state(initial_validator_deposits: List[Deposit],
         # Ethereum 1.0 chain data
         latest_eth1_data=latest_eth1_data,
         eth1_data_votes=[],
+        deposit_index=len(initial_validator_deposits)
     )
 
     # Process initial deposits
@@ -1688,7 +1676,7 @@ For each `attestation` in `block.body.attestations`:
 * Verify that `attestation.data.slot <= state.slot - MIN_ATTESTATION_INCLUSION_DELAY < attestation.data.slot + EPOCH_LENGTH`.
 * Verify that `attestation.data.justified_epoch` is equal to `state.justified_epoch if attestation.data.slot >= get_epoch_start_slot(get_current_epoch(state)) else state.previous_justified_epoch`.
 * Verify that `attestation.data.justified_block_root` is equal to `get_block_root(state, get_epoch_start_slot(attestation.data.justified_epoch))`.
-* Verify that either `attestation.data.latest_crosslink_root` or `attestation.data.shard_block_root` equals `state.latest_crosslinks[attestation.data.shard].shard_block_root`.
+* Verify that either (i) `state.latest_crosslinks[attestation.data.shard] == attestation.data.latest_crosslink` or (ii) `state.latest_crosslinks[attestation.data.shard] == Crosslink(shard_block_root=attestation.data.shard_block_root, epoch=slot_to_epoch(attestation.data.slot))`.
 * Verify bitfields and aggregate signature:
 
 ```python
@@ -1734,6 +1722,7 @@ Verify that `len(block.body.deposits) <= MAX_DEPOSITS`.
 For each `deposit` in `block.body.deposits`:
 
 * Let `serialized_deposit_data` be the serialized form of `deposit.deposit_data`. It should be 8 bytes for `deposit_data.amount` followed by 8 bytes for `deposit_data.timestamp` and then the `DepositInput` bytes. That is, it should match `deposit_data` in the [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) of which the hash was placed into the Merkle tree.
+* Verify that `deposit.index == state.deposit_index`.
 * Verify that `verify_merkle_branch(hash(serialized_deposit_data), deposit.branch, DEPOSIT_CONTRACT_TREE_DEPTH, deposit.index, state.latest_eth1_data.deposit_root)` is `True`.
 
 ```python
@@ -1762,6 +1751,8 @@ process_deposit(
 )
 ```
 
+* Set `state.deposit_index += 1`.
+
 ##### Exits
 
 Verify that `len(block.body.exits) <= MAX_EXITS`.
@@ -1779,7 +1770,7 @@ For each `exit` in `block.body.exits`:
 
 The steps below happen when `(state.slot + 1) % EPOCH_LENGTH == 0`.
 
-#### Helpers
+#### Helper variables
 
 * Let `current_epoch = get_current_epoch(state)`.
 * Let `previous_epoch = get_previous_epoch(state)`.
@@ -1860,7 +1851,7 @@ Finally, update the following:
 
 For every `slot in range(get_epoch_start_slot(previous_epoch), get_epoch_start_slot(next_epoch))`, let `crosslink_committees_at_slot = get_crosslink_committees_at_slot(state, slot)`. For every `(crosslink_committee, shard)` in `crosslink_committees_at_slot`, compute:
 
-* Set `state.latest_crosslinks[shard] = Crosslink(epoch=current_epoch, shard_block_root=winning_root(crosslink_committee))` if `3 * total_attesting_balance(crosslink_committee) >= 2 * get_total_balance(crosslink_committee)`.
+* Set `state.latest_crosslinks[shard] = Crosslink(epoch=slot_to_epoch(slot), shard_block_root=winning_root(crosslink_committee))` if `3 * total_attesting_balance(crosslink_committee) >= 2 * get_total_balance(crosslink_committee)`.
 
 #### Rewards and penalties
 

diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md
@@ -20,6 +20,7 @@ Phase 1 depends upon all of the constants defined in [Phase 0](0_beacon-chain.md
 |------------------------------|-----------------|--------|---------------|
 | `SHARD_CHUNK_SIZE`           | 2**5 (= 32)     | bytes  |               |
 | `SHARD_BLOCK_SIZE`           | 2**14 (= 16384) | bytes  |               |
+| `CROSSLINK_LOOKBACK`         | 2**5 (= 32)     | slots |               |
 | `MAX_BRANCH_CHALLENGE_DELAY` | 2**11 (= 2048)  | epochs | 9 days        |
 | `CHALLENGE_RESPONSE_DEADLINE`| 2**14 (= 16384) | epochs | 73 days       |
 | `MAX_BRANCH_CHALLENGES`      | 2**2 (= 4)      |        |               |
@@ -106,30 +107,43 @@ A node should sign a crosslink only if the following conditions hold. **If a nod
 
 First, the conditions must recursively apply to the crosslink referenced in `last_crosslink_root` for the same shard (unless `last_crosslink_root` equals zero, in which case we are at the genesis).
 
-Second, we verify the `shard_block_combined_data_root`. Let `h` be the slot _immediately after_ the slot of the shard block included by the last crosslink, and `h+n-1` be the slot number of the block directly referenced by the current `shard_block_root`. Let `B[i]` be the block at slot `h+i` in the shard chain. Let `bodies[0] .... bodies[n-1]` be the bodies of these blocks and `roots[0] ... roots[n-1]` the data roots. If there is a missing slot in the shard chain at position `h+i`, then `bodies[i] == b'\x00' * shard_block_maxbytes(state[i])` and `roots[i]` be the Merkle root of the empty data. Define `compute_merkle_root` be a simple Merkle root calculating function that takes as input a list of objects, where the list's length must be an exact power of two. We define the function for computing the combined data root as follows:
+Second, we verify the `shard_chain_commitment`.
+* Let `start_slot = state.latest_crosslinks[shard].epoch * EPOCH_LENGTH + EPOCH_LENGTH - CROSSLINK_LOOKBACK`.
+* Let `end_slot = attestation.data.slot - attestation.data.slot % EPOCH_LENGTH - CROSSLINK_LOOKBACK`.
+* Let `length = end_slot - start_slot`, `headers[0] .... headers[length-1]` be the serialized block headers in the canonical shard chain from the verifer's point of view (note that this implies that `headers` and `bodies` have been checked for validity).
+* Let `bodies[0] ... bodies[length-1]` be the bodies of the blocks.
+* Note: If there is a missing slot, then the header and body are the same as that of the block at the most recent slot that has a block.
+
+We define two helpers:
 
 ```python
-ZERO_ROOT = merkle_root(bytes([0] * SHARD_BLOCK_SIZE))
+def pad_to_power_of_2(values: List[bytes]) -> List[bytes]:
+    while not is_power_of_two(len(values)):
+        values = values + [SHARD_BLOCK_SIZE]
+    return values
+```
 
-def mk_combined_data_root(roots):
-    data = roots + [ZERO_ROOT for _ in range(len(roots), next_power_of_2(len(roots)))]
-    return compute_merkle_root(data)
+```python
+def merkle_root_of_bytes(data: bytes) -> bytes:
+    return merkle_root([data[i:i+32] for i in range(0, len(data), 32)])
 ```
 
-This outputs the root of a tree of the data roots, with the data roots all adjusted to have the same height if needed. The tree can also be viewed as a tree of all of the underlying data concatenated together, appropriately padded. Here is an equivalent definition that uses bodies instead of roots [TODO: check equivalence]:
+We define the function for computing the commitment as follows:
 
 ```python
-def mk_combined_data_root(depths, bodies):
-    data = b''.join(bodies)
-    data += bytes([0] * (next_power_of_2(len(data)) - len(data))
-    return compute_merkle_root([data[pos:pos+SHARD_CHUNK_SIZE] for pos in range(0, len(data), SHARD_CHUNK_SIZE)])
+def compute_commitment(headers: List[ShardBlock], bodies: List[bytes]) -> Bytes32:
+    return hash(
+        merkle_root(pad_to_power_of_2([merkle_root_of_bytes(zpad(serialize(h), SHARD_BLOCK_SIZE)) for h in headers])),
+        merkle_root(pad_to_power_of_2([merkle_root_of_bytes(h) for h in bodies]))
+    )
 ```
 
-Verify that the `shard_block_combined_data_root` is the output of these functions.
+The `shard_chain_commitment` is only valid if it equals `compute_commitment(headers, bodies)`.
+
 
 ### Shard block fork choice rule
 
-The fork choice rule for any shard is LMD GHOST using the validators currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (ie. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_chain_ref` is the block in the main beacon chain at the specified `slot` should be considered (if the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than a slot).
+The fork choice rule for any shard is LMD GHOST using the shard chain attestations of the persistent committee and the beacon chain attestations of the crosslink committee currently assigned to that shard, but instead of being rooted in the genesis it is rooted in the block referenced in the most recent accepted crosslink (ie. `state.crosslinks[shard].shard_block_root`). Only blocks whose `beacon_chain_ref` is the block in the main beacon chain at the specified `slot` should be considered (if the beacon chain skips a slot, then the block at that slot is considered to be the block in the beacon chain at the highest slot lower than a slot).
 
 # Updates to the beacon chain
 
@@ -285,7 +299,7 @@ Verify that `len(block.body.branch_responses) <= MAX_BRANCH_RESPONSES`.
 For each `response` in `block.body.branch_responses`:
 
 * Find the `BranchChallengeRecord` in `state.validator_registry[response.responder_index].open_branch_challenges` whose (`root`, `data_index`) match the (`root`, `data_index`) of the `response`. Verify that one such record exists (it is not possible for there to be more than one), call it `record`.
-* Verify that `verify_merkle_branch(leaf=response.data, branch=response.branch, depth=record.depth, index=record.data_index, root=record.root) is True.
+* Verify that `verify_merkle_branch(leaf=response.data, branch=response.branch, depth=record.depth, index=record.data_index, root=record.root)` is True.
 * Verify that `get_current_epoch(state) >= record.inclusion_epoch + ENTRY_EXIT_DELAY`.
 * Remove the `record` from `state.validator_registry[response.responder_index].open_branch_challenges`
 * Determine the proposer `proposer_index = get_beacon_proposer_index(state, state.slot)` and set `state.validator_balances[proposer_index] += base_reward(state, index) // MINOR_REWARD_QUOTIENT`.