diff --git a/eth2/beacon/_utils/random.py b/eth2/beacon/_utils/random.py index 364ebdb644..f0d453c81a 100644 --- a/eth2/beacon/_utils/random.py +++ b/eth2/beacon/_utils/random.py @@ -11,78 +11,93 @@ ) from eth_utils import ( to_tuple, + ValidationError, ) from eth2.beacon._utils.hash import ( hash_eth2, ) from eth2.beacon.constants import ( - RAND_BYTES, - RAND_MAX, + POWER_OF_TWO_NUMBERS, + MAX_LIST_SIZE, ) TItem = TypeVar('TItem') +def get_permuted_index(index: int, + list_size: int, + seed: Hash32, + shuffle_round_count: int=90) -> int: + """ + Return a pseudorandom permutation of `0...list_size-1` with ``seed`` as entropy. + + Utilizes 'swap or not' shuffling found in + https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf + See the 'generalized domain' algorithm on page 3. + """ + if index >= list_size: + raise ValidationError() + if list_size > MAX_LIST_SIZE: + raise ValidationError() + + for round in range(shuffle_round_count): + pivot = int.from_bytes( + hash_eth2(seed + round.to_bytes(1, 'little'))[0:8], + 'little', + ) % list_size + + flip = (pivot - index) % list_size + hash_pos = max(index, flip) + h = hash_eth2(seed + round.to_bytes(1, 'little') + (hash_pos // 256).to_bytes(4, 'little')) + byte = h[(hash_pos % 256) // 8] + bit = (byte >> (hash_pos % 8)) % 2 + index = flip if bit else index + + return index + + @to_tuple def shuffle(values: Sequence[TItem], - seed: Hash32) -> Iterable[TItem]: + seed: Hash32, + shuffle_round_count: int=90) -> Iterable[TItem]: """ - Return the shuffled ``values`` with ``seed`` as entropy. - Mainly for shuffling active validators in-protocol. + Return shuffled indices in a pseudorandom permutation `0...list_size-1` with + ``seed`` as entropy. - Spec: https://github.com/ethereum/eth2.0-specs/blob/70cef14a08de70e7bd0455d75cf380eb69694bfb/specs/core/0_beacon-chain.md#helper-functions # noqa: E501 + Utilizes 'swap or not' shuffling found in + https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf + See the 'generalized domain' algorithm on page 3. """ - values_count = len(values) - - # The range of the RNG places an upper-bound on the size of the list that - # may be shuffled. It is a logic error to supply an oversized list. - if values_count >= RAND_MAX: - raise ValueError( - "values_count (%s) should less than RAND_MAX (%s)." % - (values_count, RAND_MAX) + list_size = len(values) + + indices = list(range(list_size)) + for round in range(shuffle_round_count): + hash_bytes = b''.join( + [ + hash_eth2(seed + round.to_bytes(1, 'little') + i.to_bytes(4, 'little')) + for i in range((list_size + 255) // 256) + ] ) - output = [x for x in values] - source = seed - index = 0 - while index < values_count - 1: - # Re-hash the `source` to obtain a new pattern of bytes. - source = hash_eth2(source) - - # Iterate through the `source` bytes in 3-byte chunks. - for position in range(0, 32 - (32 % RAND_BYTES), RAND_BYTES): - # Determine the number of indices remaining in `values` and exit - # once the last index is reached. - remaining = values_count - index - if remaining == 1: - break - - # Read 3-bytes of `source` as a 24-bit little-endian integer. - sample_from_source = int.from_bytes( - source[position:position + RAND_BYTES], 'little' - ) - - # Sample values greater than or equal to `sample_max` will cause - # modulo bias when mapped into the `remaining` range. - sample_max = RAND_MAX - RAND_MAX % remaining - - # Perform a swap if the consumed entropy will not cause modulo bias. - if sample_from_source < sample_max: - # Select a replacement index for the current index. - replacement_position = (sample_from_source % remaining) + index - # Swap the current index with the replacement index. - (output[index], output[replacement_position]) = ( - output[replacement_position], - output[index] - ) - index += 1 + pivot = int.from_bytes( + hash_eth2(seed + round.to_bytes(1, 'little'))[:8], + 'little', + ) % list_size + for i in range(list_size): + flip = (pivot - indices[i]) % list_size + hash_position = indices[i] if indices[i] > flip else flip + byte = hash_bytes[hash_position // 8] + mask = POWER_OF_TWO_NUMBERS[hash_position % 8] + if byte & mask: + indices[i] = flip else: - # The sample causes modulo bias. A new sample should be read. + # not swap pass - return output + for i in indices: + yield values[i] def split(values: Sequence[TItem], split_count: int) -> Tuple[Iterable[TItem], ...]: diff --git a/eth2/beacon/committee_helpers.py b/eth2/beacon/committee_helpers.py index 1cafb65aa1..2e6b0cd127 100644 --- a/eth2/beacon/committee_helpers.py +++ b/eth2/beacon/committee_helpers.py @@ -74,7 +74,8 @@ def get_shuffling(*, epoch: Epoch, slots_per_epoch: int, target_committee_size: int, - shard_count: int) -> Tuple[Iterable[ValidatorIndex], ...]: + shard_count: int, + shuffle_round_count: int) -> Tuple[Iterable[ValidatorIndex], ...]: """ Shuffle ``validators`` into crosslink committees seeded by ``seed`` and ``epoch``. Return a list of ``committee_per_epoch`` committees where each @@ -96,7 +97,11 @@ def get_shuffling(*, ) # Shuffle - shuffled_active_validator_indices = shuffle(active_validator_indices, seed) + shuffled_active_validator_indices = shuffle( + active_validator_indices, + seed, + shuffle_round_count=shuffle_round_count, + ) # Split the shuffled list into committees_per_epoch pieces return tuple( @@ -283,6 +288,7 @@ def get_crosslink_committees_at_slot( shard_count = committee_config.SHARD_COUNT slots_per_epoch = committee_config.SLOTS_PER_EPOCH target_committee_size = committee_config.TARGET_COMMITTEE_SIZE + shuffle_round_count = committee_config.SHUFFLE_ROUND_COUNT epoch = slot_to_epoch(slot, slots_per_epoch) current_epoch = state.current_epoch(slots_per_epoch) @@ -327,6 +333,7 @@ def get_crosslink_committees_at_slot( slots_per_epoch=slots_per_epoch, target_committee_size=target_committee_size, shard_count=shard_count, + shuffle_round_count=shuffle_round_count, ) offset = slot % slots_per_epoch committees_per_slot = shuffling_context.committees_per_epoch // slots_per_epoch diff --git a/eth2/beacon/configs.py b/eth2/beacon/configs.py index da5af79667..d7a4fc645c 100644 --- a/eth2/beacon/configs.py +++ b/eth2/beacon/configs.py @@ -24,6 +24,9 @@ ('MAX_BALANCE_CHURN_QUOTIENT', int), ('BEACON_CHAIN_SHARD_NUMBER', Shard), ('MAX_INDICES_PER_SLASHABLE_VOTE', int), + ('MAX_EXIT_DEQUEUES_PER_EPOCH', int), + ('SHUFFLE_ROUND_COUNT', int), + # State list lengths ('LATEST_BLOCK_ROOTS_LENGTH', int), ('LATEST_ACTIVE_INDEX_ROOTS_LENGTH', int), ('LATEST_RANDAO_MIXES_LENGTH', int), @@ -72,6 +75,7 @@ def __init__(self, config: BeaconConfig): self.SHARD_COUNT = config.SHARD_COUNT self.SLOTS_PER_EPOCH = config.SLOTS_PER_EPOCH self.TARGET_COMMITTEE_SIZE = config.TARGET_COMMITTEE_SIZE + self.SHUFFLE_ROUND_COUNT = config.SHUFFLE_ROUND_COUNT # For seed self.MIN_SEED_LOOKAHEAD = config.MIN_SEED_LOOKAHEAD diff --git a/eth2/beacon/constants.py b/eth2/beacon/constants.py index 22c1df2ee9..0b2e321c22 100644 --- a/eth2/beacon/constants.py +++ b/eth2/beacon/constants.py @@ -8,21 +8,15 @@ ) -# -# shuffle function -# - -# The size of 3 bytes in integer -# sample_range = 2 ** (3 * 8) = 2 ** 24 = 16777216 -# sample_range = 16777216 - -# Entropy is consumed from the seed in 3-byte (24 bit) chunks. -RAND_BYTES = 3 -# The highest possible result of the RNG. -RAND_MAX = 2 ** (RAND_BYTES * 8) - 1 - EMPTY_SIGNATURE = BLSSignature(b'\x00' * 96) GWEI_PER_ETH = 10**9 FAR_FUTURE_EPOCH = Epoch(2**64 - 1) GENESIS_PARENT_ROOT = ZERO_HASH32 + +# +# shuffle function +# + +POWER_OF_TWO_NUMBERS = [1, 2, 4, 8, 16, 32, 64, 128] +MAX_LIST_SIZE = 2**40 diff --git a/eth2/beacon/state_machines/forks/serenity/configs.py b/eth2/beacon/state_machines/forks/serenity/configs.py index 04c94f9f46..d9527ef9d8 100644 --- a/eth2/beacon/state_machines/forks/serenity/configs.py +++ b/eth2/beacon/state_machines/forks/serenity/configs.py @@ -23,6 +23,9 @@ MAX_BALANCE_CHURN_QUOTIENT=2**5, # (= 32) BEACON_CHAIN_SHARD_NUMBER=Shard(2**64 - 1), MAX_INDICES_PER_SLASHABLE_VOTE=2**12, # (= 4,096) votes + MAX_EXIT_DEQUEUES_PER_EPOCH=2**2, # (= 4) + SHUFFLE_ROUND_COUNT=90, + # State list lengths LATEST_BLOCK_ROOTS_LENGTH=2**13, # (= 8,192) slots LATEST_ACTIVE_INDEX_ROOTS_LENGTH=2**13, # (= 8,192) epochs LATEST_RANDAO_MIXES_LENGTH=2**13, # (= 8,192) epochs