Merge branch 'dev' into hwwhww/mypy_ssz

Makefile

@@ -122,7 +122,7 @@ install_deposit_contract_compiler:
 
 compile_deposit_contract:
 	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \
-	python3.7 deposit_contract/compile.py contracts/validator_registration.vy
+	python3.7 deposit_contract/compile.py ../contracts/validator_registration.vy
 
 test_compile_deposit_contract:
 	cd $(DEPOSIT_CONTRACT_COMPILER_DIR); . venv/bin/activate; \

setup.py

@@ -499,7 +499,7 @@ def run(self):
         "pycryptodome==3.9.4",
         "py_ecc==2.0.0",
         "dataclasses==0.6",
-        "remerkleable==0.1.12",
+        "remerkleable==0.1.13",
         "ruamel.yaml==0.16.5",
         "lru-dict==1.1.6"
     ]

specs/phase0/p2p-interface.md

@@ -105,6 +105,7 @@ It consists of four main sections:
   - [Discovery](#discovery)
     - [Why are we using discv5 and not libp2p Kademlia DHT?](#why-are-we-using-discv5-and-not-libp2p-kademlia-dht)
     - [What is the difference between an ENR and a multiaddr, and why are we using ENRs?](#what-is-the-difference-between-an-enr-and-a-multiaddr-and-why-are-we-using-enrs)
+    - [Why do we not form ENRs and find peers until genesis block/state is known?](#why-do-we-not-form-enrs-and-find-peers-until-genesis-blockstate-is-known)
   - [Compression/Encoding](#compressionencoding)
     - [Why are we using SSZ for encoding?](#why-are-we-using-ssz-for-encoding)
     - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers)
@@ -247,6 +248,8 @@ Topics are plain UTF-8 strings and are encoded on the wire as determined by prot
 - `Name` - see table below
 - `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. See the [Encodings](#Encoding-strategies) section for further details.
 
+*Note*: `ForkDigestValue` is composed of values that are not known until the genesis block/state are available. Due to this, clients SHOULD NOT subscribe to gossipsub topics until these genesis values are known.
+
 Each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) has a maximum size of `GOSSIP_MAX_SIZE`. Clients MUST reject (fail validation) messages that are over this size limit. Likewise, clients MUST NOT emit or propagate messages larger than this limit.
 
 The `message-id` of a gossipsub message MUST be:
@@ -287,7 +290,7 @@ There are two primary global topics used to propagate beacon blocks and aggregat
 - `beacon_aggregate_and_proof` - This topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. (We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`)
     - `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` (a client MAY queue future aggregates for processing at the appropriate slot).
     - The aggregate attestation defined by `hash_tree_root(aggregate)` has _not_ already been seen (via aggregate gossip, within a verified block, or through the creation of an equivalent aggregate locally).
-    - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the slot `aggregate.data.slot`.
+    - The `aggregate` is the first valid aggregate received for the aggregator with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`.
     - The block being voted for (`aggregate.data.beacon_block_root`) passes validation.
     - `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`.
     - The aggregator's validator index is within the aggregate's committee -- i.e. `aggregate_and_proof.aggregator_index in get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)`.
@@ -316,7 +319,7 @@ Attestation subnets are used to propagate unaggregated attestations to subsectio
     - The attestation's committee index (`attestation.data.index`) is for the correct subnet.
     - `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` (a client MAY queue future attestations for processing at the appropriate slot).
     - The attestation is unaggregated -- that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit == 0b1]) == 1`).
-    - The attestation is the first valid attestation received for the participating validator for the slot, `attestation.data.slot`.
+    - There has been no other attestation seen on an attestation subnet that has an identical `attestation.data.target.epoch` and participating validator index.
     - The block being voted for (`attestation.data.beacon_block_root`) passes validation.
     - The signature of `attestation` is valid.
 
@@ -342,7 +345,9 @@ Topics are post-fixed with an encoding. Encodings define how the payload of a go
 
 #### Mainnet
 
-- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). Example: The beacon aggregate attestation topic string is `/eth2/beacon_aggregate_and_proof/ssz_snappy`, and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy.
+- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. Example: The beacon aggregate attestation topic string is `/eth2/beacon_aggregate_and_proof/ssz_snappy`, and the data field of a gossipsub message is an `AggregateAndProof` that has been SSZ-encoded and then compressed with Snappy.
+
+Snappy has two formats: "block" and "frames" (streaming). Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames.
 
 Implementations MUST use a single encoding. Changing an encoding will require coordination between participating implementations.
 
@@ -445,7 +450,7 @@ Here, `result` represents the 1-byte response code.
 The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Two values are possible at this time:
 
 -  `ssz`: the contents are [SSZ-encoded](../../ssz/simple-serialize.md). This encoding type MUST be supported by all clients. For objects containing a single field, only the field is SSZ-encoded not a container with a single field. For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s.
--  `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy). MAY be supported in the interoperability testnet; MUST be supported in mainnet.
+-  `ssz_snappy`: The contents are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) frames compression. MAY be supported in the interoperability testnet; MUST be supported in mainnet.
 
 #### SSZ-encoding strategy (with or without Snappy)
 
@@ -752,6 +757,8 @@ where the fields of `ENRForkID` are defined as
 * `next_fork_version` is the fork version corresponding to the next planned hard fork at a future epoch. If no future fork is planned, set `next_fork_version = current_fork_version` to signal this fact
 * `next_fork_epoch` is the epoch at which the next fork is planned and the `current_fork_version` will be updated. If no future fork is planned, set `next_fork_epoch = FAR_FUTURE_EPOCH` to signal this fact
 
+*Note*: `fork_digest` is composed of values that are not not known until the genesis block/state are available. Due to this, clients SHOULD NOT form ENRs and begin peer discovery until genesis values are known.
+
 Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `next_fork_epoch` that match local values.
 
 Clients MAY connect to peers with the same `fork_digest` but a different `next_fork_version`/`next_fork_epoch`. Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`.
@@ -1092,6 +1099,12 @@ discv5 uses ENRs and we will presumably need to:
 1. Add `multiaddr` to the dictionary, so that nodes can advertise their multiaddr under a reserved namespace in ENRs. – and/or –
 2. Define a bi-directional conversion function between multiaddrs and the corresponding denormalized fields in an ENR (ip, ip6, tcp, tcp6, etc.), for compatibility with nodes that do not support multiaddr natively (e.g. Eth 1.0 nodes).
 
+### Why do we not form ENRs and find peers until genesis block/state is known?
+
+Although client software might very well be running locally prior to the solidification of the eth2 genesis state and block, clients cannot form valid ENRs prior to this point. ENRs contain `fork_digest` which utilizes the `genesis_validators_root` for a cleaner separation between chains so prior to knowing genesis, we cannot use `fork_digest` to cleanly find peers on our intended chain. Once genesis data is known, we can then form ENRs and safely find peers.
+
+When using an eth1 deposit contract for deposits, `fork_digest` will be known at least `MIN_GENESIS_DELAY` (24 hours in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis.
+
 ## Compression/Encoding
 
 ### Why are we using SSZ for encoding?

tests/core/pyspec/eth2spec/config/config_util.py

@@ -8,13 +8,18 @@
 
 # Access to overwrite spec constants based on configuration
 # This is called by the spec module after declaring its globals, and applies the loaded presets.
-def apply_constants_config(spec_globals: Dict[str, Any]) -> None:
+def apply_constants_config(spec_globals: Dict[str, Any], warn_if_unknown: bool = False) -> None:
     global config
     for k, v in config.items():
-        if k.startswith('DOMAIN_'):
-            spec_globals[k] = spec_globals['DomainType'](v)  # domain types are defined as bytes in the configs
+        # the spec should have default values for everything, if not, the config key is invalid.
+        if k in spec_globals:
+            # Keep the same type as the default value indicates (which may be an SSZ basic type subclass, e.g. 'Gwei')
+            spec_globals[k] = spec_globals[k].__class__(v)
         else:
-            spec_globals[k] = v
+            # Note: Phase 0 spec will not know the phase 1 config values.
+            # Yet, during debugging you can enable explicit warnings.
+            if warn_if_unknown:
+                print(f"WARNING: unknown config key: '{k}' with value: '{v}'")
 
 
 # Load presets from a file, and then prepares the global config setting. This does not apply the config.

tests/core/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_justification_and_finalization.py

@@ -24,7 +24,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support
         raise Exception(f"cannot include attestations in epoch ${epoch} from epoch ${current_epoch}")
 
     total_balance = spec.get_total_active_balance(state)
-    remaining_balance = total_balance * 2 // 3
+    remaining_balance = int(total_balance * 2 // 3)  # can become negative
 
     start_slot = spec.compute_start_slot_at_epoch(epoch)
     for slot in range(start_slot, start_slot + spec.SLOTS_PER_EPOCH):
@@ -42,7 +42,7 @@ def add_mock_attestations(spec, state, epoch, source, target, sufficient_support
             aggregation_bits = [0] * len(committee)
             for v in range(len(committee) * 2 // 3 + 1):
                 if remaining_balance > 0:
-                    remaining_balance -= state.validators[v].effective_balance
+                    remaining_balance -= int(state.validators[v].effective_balance)
                     aggregation_bits[v] = 1
                 else:
                     break