Add get_permuted_index and new swap or not shuffle

1. Add constants `MAX_EXIT_DEQUEUES_PER_EPOCH` and `SHUFFLE_ROUND_COUNT`
2. Add Add `get_permuted_index` and new swap or not `shuffle`
3. Use the results of two different approaches to test

eth2/beacon/_utils/random.py

@@ -11,78 +11,93 @@
 )
 from eth_utils import (
     to_tuple,
+    ValidationError,
 )
 
 from eth2.beacon._utils.hash import (
     hash_eth2,
 )
 from eth2.beacon.constants import (
-    RAND_BYTES,
-    RAND_MAX,
+    POWER_OF_TWO_NUMBERS,
+    MAX_LIST_SIZE,
 )
 
 
 TItem = TypeVar('TItem')
 
 
+def get_permuted_index(index: int,
+                       list_size: int,
+                       seed: Hash32,
+                       shuffle_round_count: int=90) -> int:
+    """
+    Return a pseudorandom permutation of `0...list_size-1` with ``seed`` as entropy.
+
+    Utilizes 'swap or not' shuffling found in
+    https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
+    See the 'generalized domain' algorithm on page 3.
+    """
+    if index >= list_size:
+        raise ValidationError()
+    if list_size > MAX_LIST_SIZE:
+        raise ValidationError()
+
+    for round in range(shuffle_round_count):
+        pivot = int.from_bytes(
+            hash_eth2(seed + round.to_bytes(1, 'little'))[0:8],
+            'little',
+        ) % list_size
+
+        flip = (pivot - index) % list_size
+        hash_pos = max(index, flip)
+        h = hash_eth2(seed + round.to_bytes(1, 'little') + (hash_pos // 256).to_bytes(4, 'little'))
+        byte = h[(hash_pos % 256) // 8]
+        bit = (byte >> (hash_pos % 8)) % 2
+        index = flip if bit else index
+
+    return index
+
+
 @to_tuple
 def shuffle(values: Sequence[TItem],
-            seed: Hash32) -> Iterable[TItem]:
+            seed: Hash32,
+            shuffle_round_count: int=90) -> Iterable[TItem]:
     """
-    Return the shuffled ``values`` with ``seed`` as entropy.
-    Mainly for shuffling active validators in-protocol.
+    Return shuffled indices in a pseudorandom permutation `0...list_size-1` with
+    ``seed`` as entropy.
 
-    Spec: https://github.com/ethereum/eth2.0-specs/blob/70cef14a08de70e7bd0455d75cf380eb69694bfb/specs/core/0_beacon-chain.md#helper-functions  # noqa: E501
+    Utilizes 'swap or not' shuffling found in
+    https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
+    See the 'generalized domain' algorithm on page 3.
     """
-    values_count = len(values)
-
-    # The range of the RNG places an upper-bound on the size of the list that
-    # may be shuffled. It is a logic error to supply an oversized list.
-    if values_count >= RAND_MAX:
-        raise ValueError(
-            "values_count (%s) should less than RAND_MAX (%s)." %
-            (values_count, RAND_MAX)
+    list_size = len(values)
+
+    indices = list(range(list_size))
+    for round in range(shuffle_round_count):
+        hash_bytes = b''.join(
+            [
+                hash_eth2(seed + round.to_bytes(1, 'little') + i.to_bytes(4, 'little'))
+                for i in range((list_size + 255) // 256)
+            ]
         )
 
-    output = [x for x in values]
-    source = seed
-    index = 0
-    while index < values_count - 1:
-        # Re-hash the `source` to obtain a new pattern of bytes.
-        source = hash_eth2(source)
-
-        # Iterate through the `source` bytes in 3-byte chunks.
-        for position in range(0, 32 - (32 % RAND_BYTES), RAND_BYTES):
-            # Determine the number of indices remaining in `values` and exit
-            # once the last index is reached.
-            remaining = values_count - index
-            if remaining == 1:
-                break
-
-            # Read 3-bytes of `source` as a 24-bit little-endian integer.
-            sample_from_source = int.from_bytes(
-                source[position:position + RAND_BYTES], 'little'
-            )
-
-            # Sample values greater than or equal to `sample_max` will cause
-            # modulo bias when mapped into the `remaining` range.
-            sample_max = RAND_MAX - RAND_MAX % remaining
-
-            # Perform a swap if the consumed entropy will not cause modulo bias.
-            if sample_from_source < sample_max:
-                # Select a replacement index for the current index.
-                replacement_position = (sample_from_source % remaining) + index
-                # Swap the current index with the replacement index.
-                (output[index], output[replacement_position]) = (
-                    output[replacement_position],
-                    output[index]
-                )
-                index += 1
+        pivot = int.from_bytes(
+            hash_eth2(seed + round.to_bytes(1, 'little'))[:8],
+            'little',
+        ) % list_size
+        for i in range(list_size):
+            flip = (pivot - indices[i]) % list_size
+            hash_position = indices[i] if indices[i] > flip else flip
+            byte = hash_bytes[hash_position // 8]
+            mask = POWER_OF_TWO_NUMBERS[hash_position % 8]
+            if byte & mask:
+                indices[i] = flip
             else:
-                # The sample causes modulo bias. A new sample should be read.
+                # not swap
                 pass
 
-    return output
+    for i in indices:
+        yield values[i]
 
 
 def split(values: Sequence[TItem], split_count: int) -> Tuple[Iterable[TItem], ...]:

eth2/beacon/committee_helpers.py

@@ -74,7 +74,8 @@ def get_shuffling(*,
                   epoch: Epoch,
                   slots_per_epoch: int,
                   target_committee_size: int,
-                  shard_count: int) -> Tuple[Iterable[ValidatorIndex], ...]:
+                  shard_count: int,
+                  shuffle_round_count: int) -> Tuple[Iterable[ValidatorIndex], ...]:
     """
     Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``.
     Return a list of ``committee_per_epoch`` committees where each
@@ -96,7 +97,11 @@ def get_shuffling(*,
     )
 
     # Shuffle
-    shuffled_active_validator_indices = shuffle(active_validator_indices, seed)
+    shuffled_active_validator_indices = shuffle(
+        active_validator_indices,
+        seed,
+        shuffle_round_count=shuffle_round_count,
+    )
 
     # Split the shuffled list into committees_per_epoch pieces
     return tuple(
@@ -283,6 +288,7 @@ def get_crosslink_committees_at_slot(
     shard_count = committee_config.SHARD_COUNT
     slots_per_epoch = committee_config.SLOTS_PER_EPOCH
     target_committee_size = committee_config.TARGET_COMMITTEE_SIZE
+    shuffle_round_count = committee_config.SHUFFLE_ROUND_COUNT
 
     epoch = slot_to_epoch(slot, slots_per_epoch)
     current_epoch = state.current_epoch(slots_per_epoch)
@@ -327,6 +333,7 @@ def get_crosslink_committees_at_slot(
         slots_per_epoch=slots_per_epoch,
         target_committee_size=target_committee_size,
         shard_count=shard_count,
+        shuffle_round_count=shuffle_round_count,
     )
     offset = slot % slots_per_epoch
     committees_per_slot = shuffling_context.committees_per_epoch // slots_per_epoch

eth2/beacon/configs.py

@@ -24,6 +24,9 @@
         ('MAX_BALANCE_CHURN_QUOTIENT', int),
         ('BEACON_CHAIN_SHARD_NUMBER', Shard),
         ('MAX_INDICES_PER_SLASHABLE_VOTE', int),
+        ('MAX_EXIT_DEQUEUES_PER_EPOCH', int),
+        ('SHUFFLE_ROUND_COUNT', int),
+        # State list lengths
         ('LATEST_BLOCK_ROOTS_LENGTH', int),
         ('LATEST_ACTIVE_INDEX_ROOTS_LENGTH', int),
         ('LATEST_RANDAO_MIXES_LENGTH', int),
@@ -72,6 +75,7 @@ def __init__(self, config: BeaconConfig):
         self.SHARD_COUNT = config.SHARD_COUNT
         self.SLOTS_PER_EPOCH = config.SLOTS_PER_EPOCH
         self.TARGET_COMMITTEE_SIZE = config.TARGET_COMMITTEE_SIZE
+        self.SHUFFLE_ROUND_COUNT = config.SHUFFLE_ROUND_COUNT
 
         # For seed
         self.MIN_SEED_LOOKAHEAD = config.MIN_SEED_LOOKAHEAD

eth2/beacon/constants.py

@@ -8,21 +8,15 @@
 )
 
 
-#
-# shuffle function
-#
-
-# The size of 3 bytes in integer
-# sample_range = 2 ** (3 * 8) = 2 ** 24 = 16777216
-# sample_range = 16777216
-
-# Entropy is consumed from the seed in 3-byte (24 bit) chunks.
-RAND_BYTES = 3
-# The highest possible result of the RNG.
-RAND_MAX = 2 ** (RAND_BYTES * 8) - 1
-
 EMPTY_SIGNATURE = BLSSignature(b'\x00' * 96)
 GWEI_PER_ETH = 10**9
 FAR_FUTURE_EPOCH = Epoch(2**64 - 1)
 
 GENESIS_PARENT_ROOT = ZERO_HASH32
+
+#
+# shuffle function
+#
+
+POWER_OF_TWO_NUMBERS = [1, 2, 4, 8, 16, 32, 64, 128]
+MAX_LIST_SIZE = 2**40

eth2/beacon/state_machines/forks/serenity/configs.py

@@ -23,6 +23,9 @@
     MAX_BALANCE_CHURN_QUOTIENT=2**5,  # (= 32)
     BEACON_CHAIN_SHARD_NUMBER=Shard(2**64 - 1),
     MAX_INDICES_PER_SLASHABLE_VOTE=2**12,  # (= 4,096) votes
+    MAX_EXIT_DEQUEUES_PER_EPOCH=2**2,  # (= 4)
+    SHUFFLE_ROUND_COUNT=90,
+    # State list lengths
     LATEST_BLOCK_ROOTS_LENGTH=2**13,  # (= 8,192) slots
     LATEST_ACTIVE_INDEX_ROOTS_LENGTH=2**13,  # (= 8,192) epochs
     LATEST_RANDAO_MIXES_LENGTH=2**13,  # (= 8,192) epochs