Add get_permuted_index and new swap or not shuffle

1. Add constants `MAX_EXIT_DEQUEUES_PER_EPOCH` and `SHUFFLE_ROUND_COUNT`
2. Add `get_permuted_index` and new swap or not `shuffle`
3. Use the results of two different approaches to test

eth2/beacon/_utils/random.py

@@ -11,78 +11,93 @@
 )
 from eth_utils import (
     to_tuple,
+    ValidationError,
 )
 
 from eth2.beacon._utils.hash import (
     hash_eth2,
 )
 from eth2.beacon.constants import (
-    RAND_BYTES,
-    RAND_MAX,
+    POWER_OF_TWO_NUMBERS,
+    MAX_LIST_SIZE,
 )
 
 
 TItem = TypeVar('TItem')
 
 
+def get_permuted_index(index: int,
+                       list_size: int,
+                       seed: Hash32,
+                       shuffle_round_count: int=90) -> int:
+    """
+    Return a pseudorandom permutation of `0...list_size-1` with ``seed`` as entropy.
+
+    Utilizes 'swap or not' shuffling found in
+    https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
+    See the 'generalized domain' algorithm on page 3.
+    """
+    if index >= list_size:
+        raise ValidationError()
+    if list_size > MAX_LIST_SIZE:
+        raise ValidationError()
+
+    for round in range(shuffle_round_count):
+        pivot = int.from_bytes(
+            hash_eth2(seed + round.to_bytes(1, 'little'))[0:8],
+            'little',
+        ) % list_size
+
+        flip = (pivot - index) % list_size
+        hash_pos = max(index, flip)
+        h = hash_eth2(seed + round.to_bytes(1, 'little') + (hash_pos // 256).to_bytes(4, 'little'))
+        byte = h[(hash_pos % 256) // 8]
+        bit = (byte >> (hash_pos % 8)) % 2
+        index = flip if bit else index
+
+    return index
+
+
 @to_tuple
 def shuffle(values: Sequence[TItem],
-            seed: Hash32) -> Iterable[TItem]:
+            seed: Hash32,
+            shuffle_round_count: int=90) -> Iterable[TItem]:
     """
-    Return the shuffled ``values`` with ``seed`` as entropy.
-    Mainly for shuffling active validators in-protocol.
+    Return shuffled indices in a pseudorandom permutation `0...list_size-1` with
+    ``seed`` as entropy.
 
-    Spec: https://github.com/ethereum/eth2.0-specs/blob/70cef14a08de70e7bd0455d75cf380eb69694bfb/specs/core/0_beacon-chain.md#helper-functions  # noqa: E501
+    Utilizes 'swap or not' shuffling found in
+    https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
+    See the 'generalized domain' algorithm on page 3.
     """
-    values_count = len(values)
-
-    # The range of the RNG places an upper-bound on the size of the list that
-    # may be shuffled. It is a logic error to supply an oversized list.
-    if values_count >= RAND_MAX:
-        raise ValueError(
-            "values_count (%s) should less than RAND_MAX (%s)." %
-            (values_count, RAND_MAX)
+    list_size = len(values)
+
+    indices = list(range(list_size))
+    for round in range(shuffle_round_count):
+        hash_bytes = b''.join(
+            [
+                hash_eth2(seed + round.to_bytes(1, 'little') + i.to_bytes(4, 'little'))
+                for i in range((list_size + 255) // 256)
+            ]
         )
 
-    output = [x for x in values]
-    source = seed
-    index = 0
-    while index < values_count - 1:
-        # Re-hash the `source` to obtain a new pattern of bytes.
-        source = hash_eth2(source)
-
-        # Iterate through the `source` bytes in 3-byte chunks.
-        for position in range(0, 32 - (32 % RAND_BYTES), RAND_BYTES):
-            # Determine the number of indices remaining in `values` and exit
-            # once the last index is reached.
-            remaining = values_count - index
-            if remaining == 1:
-                break
-
-            # Read 3-bytes of `source` as a 24-bit little-endian integer.
-            sample_from_source = int.from_bytes(
-                source[position:position + RAND_BYTES], 'little'
-            )
-
-            # Sample values greater than or equal to `sample_max` will cause
-            # modulo bias when mapped into the `remaining` range.
-            sample_max = RAND_MAX - RAND_MAX % remaining
-
-            # Perform a swap if the consumed entropy will not cause modulo bias.
-            if sample_from_source < sample_max:
-                # Select a replacement index for the current index.
-                replacement_position = (sample_from_source % remaining) + index
-                # Swap the current index with the replacement index.
-                (output[index], output[replacement_position]) = (
-                    output[replacement_position],
-                    output[index]
-                )
-                index += 1
+        pivot = int.from_bytes(
+            hash_eth2(seed + round.to_bytes(1, 'little'))[:8],
+            'little',
+        ) % list_size
+        for i in range(list_size):
+            flip = (pivot - indices[i]) % list_size
+            hash_position = indices[i] if indices[i] > flip else flip
+            byte = hash_bytes[hash_position // 8]
+            mask = POWER_OF_TWO_NUMBERS[hash_position % 8]
+            if byte & mask:
+                indices[i] = flip
             else:
-                # The sample causes modulo bias. A new sample should be read.
+                # not swap
                 pass
 
-    return output
+    for i in indices:
+        yield values[i]
 
 
 def split(values: Sequence[TItem], split_count: int) -> Tuple[Iterable[TItem], ...]:

eth2/beacon/committee_helpers.py

@@ -74,7 +74,8 @@ def get_shuffling(*,
                   epoch: Epoch,
                   slots_per_epoch: int,
                   target_committee_size: int,
-                  shard_count: int) -> Tuple[Iterable[ValidatorIndex], ...]:
+                  shard_count: int,
+                  shuffle_round_count: int) -> Tuple[Iterable[ValidatorIndex], ...]:
     """
     Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``.
     Return a list of ``committee_per_epoch`` committees where each
@@ -96,7 +97,11 @@ def get_shuffling(*,
     )
 
     # Shuffle
-    shuffled_active_validator_indices = shuffle(active_validator_indices, seed)
+    shuffled_active_validator_indices = shuffle(
+        active_validator_indices,
+        seed,
+        shuffle_round_count=shuffle_round_count,
+    )
 
     # Split the shuffled list into committees_per_epoch pieces
     return tuple(
@@ -283,6 +288,7 @@ def get_crosslink_committees_at_slot(
     shard_count = committee_config.SHARD_COUNT
     slots_per_epoch = committee_config.SLOTS_PER_EPOCH
     target_committee_size = committee_config.TARGET_COMMITTEE_SIZE
+    shuffle_round_count = committee_config.SHUFFLE_ROUND_COUNT
 
     epoch = slot_to_epoch(slot, slots_per_epoch)
     current_epoch = state.current_epoch(slots_per_epoch)
@@ -327,6 +333,7 @@ def get_crosslink_committees_at_slot(
         slots_per_epoch=slots_per_epoch,
         target_committee_size=target_committee_size,
         shard_count=shard_count,
+        shuffle_round_count=shuffle_round_count,
     )
     offset = slot % slots_per_epoch
     committees_per_slot = shuffling_context.committees_per_epoch // slots_per_epoch

eth2/beacon/configs.py

@@ -24,6 +24,9 @@
         ('MAX_BALANCE_CHURN_QUOTIENT', int),
         ('BEACON_CHAIN_SHARD_NUMBER', Shard),
         ('MAX_INDICES_PER_SLASHABLE_VOTE', int),
+        ('MAX_EXIT_DEQUEUES_PER_EPOCH', int),
+        ('SHUFFLE_ROUND_COUNT', int),
+        # State list lengths
         ('LATEST_BLOCK_ROOTS_LENGTH', int),
         ('LATEST_ACTIVE_INDEX_ROOTS_LENGTH', int),
         ('LATEST_RANDAO_MIXES_LENGTH', int),
@@ -72,6 +75,7 @@ def __init__(self, config: BeaconConfig):
         self.SHARD_COUNT = config.SHARD_COUNT
         self.SLOTS_PER_EPOCH = config.SLOTS_PER_EPOCH
         self.TARGET_COMMITTEE_SIZE = config.TARGET_COMMITTEE_SIZE
+        self.SHUFFLE_ROUND_COUNT = config.SHUFFLE_ROUND_COUNT
 
         # For seed
         self.MIN_SEED_LOOKAHEAD = config.MIN_SEED_LOOKAHEAD

eth2/beacon/constants.py

@@ -8,21 +8,15 @@
 )
 
 
-#
-# shuffle function
-#
-
-# The size of 3 bytes in integer
-# sample_range = 2 ** (3 * 8) = 2 ** 24 = 16777216
-# sample_range = 16777216
-
-# Entropy is consumed from the seed in 3-byte (24 bit) chunks.
-RAND_BYTES = 3
-# The highest possible result of the RNG.
-RAND_MAX = 2 ** (RAND_BYTES * 8) - 1
-
 EMPTY_SIGNATURE = BLSSignature(b'\x00' * 96)
 GWEI_PER_ETH = 10**9
 FAR_FUTURE_EPOCH = Epoch(2**64 - 1)
 
 GENESIS_PARENT_ROOT = ZERO_HASH32
+
+#
+# shuffle function
+#
+
+POWER_OF_TWO_NUMBERS = [1, 2, 4, 8, 16, 32, 64, 128]
+MAX_LIST_SIZE = 2**40

eth2/beacon/state_machines/forks/serenity/configs.py

@@ -23,6 +23,9 @@
     MAX_BALANCE_CHURN_QUOTIENT=2**5,  # (= 32)
     BEACON_CHAIN_SHARD_NUMBER=Shard(2**64 - 1),
     MAX_INDICES_PER_SLASHABLE_VOTE=2**12,  # (= 4,096) votes
+    MAX_EXIT_DEQUEUES_PER_EPOCH=2**2,  # (= 4)
+    SHUFFLE_ROUND_COUNT=90,
+    # State list lengths
     LATEST_BLOCK_ROOTS_LENGTH=2**13,  # (= 8,192) slots
     LATEST_ACTIVE_INDEX_ROOTS_LENGTH=2**13,  # (= 8,192) epochs
     LATEST_RANDAO_MIXES_LENGTH=2**13,  # (= 8,192) epochs

tests/eth2/beacon/_utils/test_random.py

@@ -1,27 +1,31 @@
 import pytest
 
 from eth2.beacon._utils.random import (
+    get_permuted_index,
     shuffle,
 )
 
 
+def slow_shuffle(items, seed):
+    length = len(items)
+    return tuple([items[get_permuted_index(i, length, seed)] for i in range(length)])
+
+
 @pytest.mark.parametrize(
     (
-        'values,seed,expect'
+        'values,seed'
     ),
     [
         (
             tuple(range(12)),
             b'\x23' * 32,
-            (8, 3, 9, 0, 1, 11, 2, 4, 6, 7, 10, 5),
+        ),
+        (
+            tuple(range(2**6))[10:],
+            b'\x67' * 32,
         ),
     ],
 )
-def test_shuffle_consistent(values, seed, expect):
+def test_shuffle_consistent(values, seed):
+    expect = slow_shuffle(values, seed)
     assert shuffle(values, seed) == expect
-
-
-def test_shuffle_out_of_bound():
-    values = [i for i in range(2**24 + 1)]
-    with pytest.raises(ValueError):
-        shuffle(values, b'hello')

tests/eth2/beacon/conftest.py

@@ -433,6 +433,16 @@ def max_indices_per_slashable_vote():
     return SERENITY_CONFIG.MAX_INDICES_PER_SLASHABLE_VOTE
 
 
+@pytest.fixture
+def max_exit_dequeues_per_epoch():
+    return SERENITY_CONFIG.MAX_EXIT_DEQUEUES_PER_EPOCH
+
+
+@pytest.fixture
+def shuffle_round_count():
+    return SERENITY_CONFIG.SHUFFLE_ROUND_COUNT
+
+
 @pytest.fixture
 def latest_block_roots_length():
     return SERENITY_CONFIG.LATEST_BLOCK_ROOTS_LENGTH
@@ -672,6 +682,8 @@ def config(
         max_balance_churn_quotient,
         beacon_chain_shard_number,
         max_indices_per_slashable_vote,
+        max_exit_dequeues_per_epoch,
+        shuffle_round_count,
         latest_block_roots_length,
         latest_active_index_roots_length,
         latest_randao_mixes_length,
@@ -710,6 +722,8 @@ def config(
         MAX_BALANCE_CHURN_QUOTIENT=max_balance_churn_quotient,
         BEACON_CHAIN_SHARD_NUMBER=beacon_chain_shard_number,
         MAX_INDICES_PER_SLASHABLE_VOTE=max_indices_per_slashable_vote,
+        MAX_EXIT_DEQUEUES_PER_EPOCH=max_exit_dequeues_per_epoch,
+        SHUFFLE_ROUND_COUNT=shuffle_round_count,
         LATEST_BLOCK_ROOTS_LENGTH=latest_block_roots_length,
         LATEST_ACTIVE_INDEX_ROOTS_LENGTH=latest_active_index_roots_length,
         LATEST_RANDAO_MIXES_LENGTH=latest_randao_mixes_length,

tests/eth2/beacon/state_machines/forks/test_serenity_epoch_processing.py

@@ -589,19 +589,20 @@ def test_process_rewards_and_penalties_for_finality(
             10,
             4,
             40,
-            {2, 3, 4, 5, 6, 9, 10, 11, 12, 13, 15, 16, 17},
+            {2, 3, 4, 5, 6, 7, 9, 10, 11, 12, 13, 15, 16, 17},
             {
                 2: 31,  # proposer index for inclusion slot 31: 6
                 3: 31,
-                4: 32,  # proposer index for inclusion slot 32: 12
+                4: 32,  # proposer index for inclusion slot 32: 16
                 5: 32,
                 6: 32,
+                7: 32,
                 9: 35,  # proposer index for inclusion slot 35: 19
                 10: 35,
                 11: 35,
                 12: 35,
                 13: 35,
-                15: 38,  # proposer index for inclusion slot 38: 8
+                15: 38,  # proposer index for inclusion slot 38: 15
                 16: 38,
                 17: 38,
             },
@@ -615,15 +616,15 @@ def test_process_rewards_and_penalties_for_finality(
                 5: 0,
                 6: 50,  # 2 * (100 // 4)
                 7: 0,
-                8: 75,  # 3 * (100 // 4)
+                8: 0,
                 9: 0,
                 10: 0,
                 11: 0,
-                12: 75,  # 3 * (100 // 4)
+                12: 0,
                 13: 0,
                 14: 0,
-                15: 0,
-                16: 0,
+                15: 75,  # 3 * (100 // 4)
+                16: 100,  # 4 * (100 // 4)
                 17: 0,
                 18: 0,
                 19: 125,  # 5 * (100 // 4)

tests/eth2/beacon/test_committee_helpers.py

@@ -140,6 +140,7 @@ def test_get_next_epoch_committee_count(n_validators_state,
 def test_get_shuffling_is_complete(activated_genesis_validators,
                                    slots_per_epoch,
                                    target_committee_size,
+                                   shuffle_round_count,
                                    shard_count,
                                    epoch):
     shuffling = get_shuffling(
@@ -149,6 +150,7 @@ def test_get_shuffling_is_complete(activated_genesis_validators,
         slots_per_epoch=slots_per_epoch,
         target_committee_size=target_committee_size,
         shard_count=shard_count,
+        shuffle_round_count=shuffle_round_count,
     )
 
     assert len(shuffling) == slots_per_epoch
@@ -322,6 +324,7 @@ def test_get_crosslink_committees_at_slot(
         slots_per_epoch,
         target_committee_size,
         shard_count,
+        shuffle_round_count,
         genesis_epoch,
         committee_config,
         registry_change,
@@ -428,6 +431,7 @@ def mock_generate_seed(state,
         slots_per_epoch=slots_per_epoch,
         target_committee_size=target_committee_size,
         shard_count=shard_count,
+        shuffle_round_count=shuffle_round_count,
     )
     assert shuffling[committees_per_slot * offset] == crosslink_committees_at_slot[0][0]