Merge branch 'unstable' into extra-files

.github/workflows/ci.yml

@@ -215,7 +215,7 @@ jobs:
         if: ${{ !cancelled() }} && github.event_name == 'pull_request'
         run: |
           excluded_files="config.yaml"
-          excluded_extensions="ans|json|md|png|ssz|txt"
+          excluded_extensions="ans|json|md|png|service|ssz|txt"
 
           current_year=$(date +"%Y")
           outdated_files=()

.gitmodules

@@ -24,7 +24,7 @@
 	path = vendor/nim-libp2p
 	url = https://github.com/status-im/nim-libp2p.git
 	ignore = untracked
-	branch = nimbus
+	branch = unstable
 [submodule "vendor/nimbus-build-system"]
 	path = vendor/nimbus-build-system
 	url = https://github.com/status-im/nimbus-build-system.git
@@ -59,7 +59,7 @@
 	path = vendor/nim-chronos
 	url = https://github.com/status-im/nim-chronos.git
 	ignore = untracked
-	branch = nimbus-v23.9.0
+	branch = master
 [submodule "vendor/nim-chronicles"]
 	path = vendor/nim-chronicles
 	url = https://github.com/status-im/nim-chronicles.git
@@ -222,6 +222,6 @@
 	branch = master
 [submodule "vendor/holesky"]
 	path = vendor/holesky
-  url = https://github.com/eth-clients/holesky
-  ignore = untracked
-  branch = main
+	url = https://github.com/eth-clients/holesky
+	ignore = untracked
+	branch = main

Makefile

@@ -118,6 +118,10 @@ ifneq ($(OS), Windows_NT)
 PLATFORM_SPECIFIC_TARGETS += gnosis-build
 endif
 
+# We don't need the `vendor/holesky/public-keys/all.txt` file but fetching it
+# may trigger 'This repository is over its data quota' from GitHub
+GIT_SUBMODULE_CONFIG := -c lfs.fetchexclude=/public-keys/all.txt
+
 ifeq ($(NIM_PARAMS),)
 # "variables.mk" was not included, so we update the submodules.
 #
@@ -134,11 +138,11 @@ $(error Git LFS not installed)
 endif
 endif
 
-GIT_SUBMODULE_UPDATE := git submodule update --init --recursive
+GIT_SUBMODULE_UPDATE := git $(GIT_SUBMODULE_CONFIG) submodule update --init --recursive
 .DEFAULT:
 	+@ echo -e "Git submodules not found. Running '$(GIT_SUBMODULE_UPDATE)'.\n"; \
 		$(GIT_SUBMODULE_UPDATE) && \
-		git submodule foreach --quiet 'git reset --quiet --hard' && \
+		git submodule foreach --quiet 'git $(GIT_SUBMODULE_CONFIG) reset --quiet --hard' && \
 		echo
 # Now that the included *.mk files appeared, and are newer than this file, Make will restart itself:
 # https://www.gnu.org/software/make/manual/make.html#Remaking-Makefiles

beacon_chain/conf.nim

@@ -551,7 +551,7 @@ type
         name: "dump" .}: bool
 
       directPeers* {.
-        desc: "The list of privileged, secure and known peers to connect and maintain the connection to, this requires a not random netkey-file. In the complete multiaddress format like: /ip4/<address>/tcp/<port>/p2p/<peerId-public-key>. Peering agreements are established out of band and must be reciprocal."
+        desc: "The list of privileged, secure and known peers to connect and maintain the connection to. This requires a not random netkey-file. In the multiaddress format like: /ip4/<address>/tcp/<port>/p2p/<peerId-public-key>, or enr format (enr:-xx). Peering agreements are established out of band and must be reciprocal"
         name: "direct-peer" .}: seq[string]
 
       doppelgangerDetection* {.

beacon_chain/consensus_object_pools/README.md

@@ -7,8 +7,8 @@ to specs:
 - blocks: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#beacon_block
 - aggregate attestations: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#beacon_aggregate_and_proof
 - unaggregated attestation: https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/p2p-interface.md#beacon_attestation_subnet_id
-- voluntary exits: https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.1/specs/phase0/p2p-interface.md#voluntary_exit
-- Attester slashings: https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/p2p-interface.md#attester_slashing
+- voluntary exits: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#voluntary_exit
+- Attester slashings: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#attester_slashing
 - Proposer slashings: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#proposer_slashing
 
 After "gossip validation" the consensus objects can be rebroadcasted as they are optimistically good, however for internal processing further verification is needed.

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -560,7 +560,7 @@ proc getAttestationsForBlock*(pool: var AttestationPool,
                               cache: var StateCache): seq[Attestation] =
   ## Retrieve attestations that may be added to a new block at the slot of the
   ## given state
-  ## https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#attestations
+  ## https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#attestations
   let newBlockSlot = state.data.slot.uint64
 
   if newBlockSlot < MIN_ATTESTATION_INCLUSION_DELAY:
@@ -743,7 +743,7 @@ func getAggregatedAttestation*(pool: var AttestationPool,
                                index: CommitteeIndex): Opt[Attestation] =
   ## Select the attestation that has the most votes going for it in the given
   ## slot/index
-  ## https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/validator.md#construct-aggregate
+  ## https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/validator.md#construct-aggregate
   let candidateIdx = pool.candidateIdx(slot)
   if candidateIdx.isNone:
     return Opt.none(Attestation)

beacon_chain/fork_choice/README.md

@@ -1,5 +1,5 @@
 # Fork choice implementations
 
 References:
-- https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/fork-choice.md
+- https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/fork-choice.md
 - https://github.com/protolambda/lmd-ghost

beacon_chain/gossip_processing/README.md

@@ -9,11 +9,11 @@ This folder holds a collection of modules to:
 
 Gossip validation is different from consensus verification in particular for blocks.
 
-- Blocks: https://github.com/ethereum/consensus-specs/blob/v1.3.0/specs/phase0/p2p-interface.md#beacon_block
+- Blocks: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#beacon_block
 - Attestations (aggregated): https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#beacon_aggregate_and_proof
 - Attestations (unaggregated): https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#attestation-subnets
 - Voluntary exits: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#voluntary_exit
-- Proposer slashings: https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/p2p-interface.md#proposer_slashing
+- Proposer slashings: https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/p2p-interface.md#proposer_slashing
 - Attester slashing: https://github.com/ethereum/consensus-specs/blob/v1.4.0-alpha.3/specs/phase0/p2p-interface.md#attester_slashing
 
 There are multiple consumers of validated consensus objects:

beacon_chain/libnimbus_lc/libnimbus_lc.h

@@ -94,7 +94,7 @@ typedef struct ETHConsensusConfig ETHConsensusConfig;
  *         based on the given `config.yaml` file content - If successful.
  * @return `NULL` - If the given `config.yaml` is malformed or incompatible.
  *
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/configs/README.md
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/configs/README.md
  */
 ETH_RESULT_USE_CHECK
 ETHConsensusConfig *ETHConsensusConfigCreateFromYaml(const char *configFileContent);
@@ -152,8 +152,8 @@ typedef struct ETHBeaconState ETHBeaconState;
  * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/beacon-chain.md#beaconstate
  * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/beacon-chain.md#beaconstate
  * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/bellatrix/beacon-chain.md#beaconstate
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/capella/beacon-chain.md#beaconstate
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/configs/README.md
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/capella/beacon-chain.md#beaconstate
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/configs/README.md
  */
 ETH_RESULT_USE_CHECK
 ETHBeaconState *ETHBeaconStateCreateFromSsz(
@@ -198,7 +198,7 @@ ETHRoot *ETHBeaconStateCopyGenesisValidatorsRoot(const ETHBeaconState *state);
  *
  * @param      root                 Merkle root.
  *
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/phase0/beacon-chain.md#custom-types
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/beacon-chain.md#custom-types
  */
 void ETHRootDestroy(ETHRoot *root);
 
@@ -576,7 +576,7 @@ typedef struct ETHLightClientHeader ETHLightClientHeader;
  *
  * @return Latest finalized header.
  *
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/capella/light-client/sync-protocol.md#modified-lightclientheader
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/capella/light-client/sync-protocol.md#modified-lightclientheader
  */
 ETH_RESULT_USE_CHECK
 const ETHLightClientHeader *ETHLightClientStoreGetFinalizedHeader(
@@ -595,7 +595,7 @@ const ETHLightClientHeader *ETHLightClientStoreGetFinalizedHeader(
  * @return Whether or not the next sync committee is currently known.
  *
  * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/light-client/sync-protocol.md#is_next_sync_committee_known
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/altair/light-client/light-client.md
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/altair/light-client/light-client.md
  */
 ETH_RESULT_USE_CHECK
 bool ETHLightClientStoreIsNextSyncCommitteeKnown(const ETHLightClientStore *store);
@@ -669,7 +669,7 @@ void ETHLightClientHeaderDestroy(ETHLightClientHeader *header);
  *
  * @return Pointer to a copy of the given header's beacon block root.
  *
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/phase0/beacon-chain.md#hash_tree_root
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/beacon-chain.md#hash_tree_root
  */
 ETH_RESULT_USE_CHECK
 ETHRoot *ETHLightClientHeaderCopyBeaconRoot(
@@ -692,7 +692,7 @@ typedef struct ETHBeaconBlockHeader ETHBeaconBlockHeader;
  *
  * @return Beacon block header.
  *
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/phase0/beacon-chain.md#beaconblockheader
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/phase0/beacon-chain.md#beaconblockheader
  */
 ETH_RESULT_USE_CHECK
 const ETHBeaconBlockHeader *ETHLightClientHeaderGetBeacon(
@@ -795,7 +795,7 @@ typedef struct ETHExecutionPayloadHeader ETHExecutionPayloadHeader;
  *
  * @return Execution payload header.
  *
- * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/deneb/beacon-chain.md#executionpayloadheader
+ * @see https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#executionpayloadheader
  */
 ETH_RESULT_USE_CHECK
 const ETHExecutionPayloadHeader *ETHLightClientHeaderGetExecution(

beacon_chain/libnimbus_lc/libnimbus_lc.nim

@@ -947,7 +947,7 @@ proc ETHLightClientHeaderCopyExecutionHash(
   ## * Pointer to a copy of the given header's execution block hash.
   ##
   ## See:
-  ## * https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/deneb/beacon-chain.md#executionpayloadheader
+  ## * https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#executionpayloadheader
   discard cfg  # Future-proof against SSZ execution block header, EIP-6404ff.
   let root = Eth2Digest.new()
   root[] = header[].execution.block_hash
@@ -972,7 +972,7 @@ func ETHLightClientHeaderGetExecution(
   ## * Execution payload header.
   ##
   ## See:
-  ## * https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.0/specs/deneb/beacon-chain.md#executionpayloadheader
+  ## * https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.1/specs/deneb/beacon-chain.md#executionpayloadheader
   addr header[].execution
 
 func ETHExecutionPayloadHeaderGetParentHash(

beacon_chain/networking/eth2_network.nim

@@ -48,6 +48,8 @@ type
   ErrorMsg = List[byte, 256]
   SendResult* = Result[void, cstring]
 
+  DirectPeers = Table[PeerId, seq[MultiAddress]]
+
   # TODO: This is here only to eradicate a compiler
   # warning about unused import (rpc/messages).
   GossipMsg = messages.Message
@@ -77,6 +79,7 @@ type
     forkDigests*: ref ForkDigests
     rng*: ref HmacDrbgContext
     peers*: Table[PeerId, Peer]
+    directPeers*: DirectPeers
     validTopics: HashSet[string]
     peerPingerHeartbeatFut: Future[void]
     peerTrimmerHeartbeatFut: Future[void]
@@ -1509,6 +1512,7 @@ proc trimConnections(node: Eth2Node, count: int) =
   var toKick = count
 
   for peerId in scores.keys:
+    if peerId in node.directPeers: continue
     debug "kicking peer", peerId, score=scores[peerId]
     asyncSpawn node.getPeer(peerId).disconnect(PeerScoreLow)
     dec toKick
@@ -1793,6 +1797,7 @@ proc new(T: type Eth2Node,
          switch: Switch, pubsub: GossipSub,
          ip: Option[ValidIpAddress], tcpPort, udpPort: Option[Port],
          privKey: keys.PrivateKey, discovery: bool,
+         directPeers: DirectPeers,
          rng: ref HmacDrbgContext): T {.raises: [CatchableError].} =
   when not defined(local_testnet):
     let
@@ -1835,6 +1840,7 @@ proc new(T: type Eth2Node,
     rng: rng,
     connectTimeout: connectTimeout,
     seenThreshold: seenThreshold,
+    directPeers: directPeers,
     quota: TokenBucket.new(maxGlobalQuota, fullReplenishTime)
   )
 
@@ -2313,6 +2319,24 @@ proc createEth2Node*(rng: ref HmacDrbgContext,
     except CatchableError as exc: raise exc
     except Exception as exc: raiseAssert exc.msg
 
+    directPeers = block:
+      var res: DirectPeers
+      for s in config.directPeers:
+        let (peerId, address) =
+          if s.startsWith("enr:"):
+            let
+              typedEnr = parseBootstrapAddress(s).get().toTypedRecord().get()
+              peerAddress = toPeerAddr(typedEnr, tcpProtocol).get()
+            (peerAddress.peerId, peerAddress.addrs[0])
+          elif s.startsWith("/"):
+            parseFullAddress(s).tryGet()
+          else:
+            fatal "direct peers address should start with / (multiaddress) or enr:", conf=s
+            quit 1
+        res.mgetOrPut(peerId, @[]).add(address)
+        info "Adding privileged direct peer", peerId, address
+      res
+
     hostAddress = tcpEndPoint(config.listenAddress, config.tcpPort)
     announcedAddresses = if extIp.isNone() or extTcpPort.isNone(): @[]
                          else: @[tcpEndPoint(extIp.get(), extTcpPort.get())]
@@ -2371,19 +2395,7 @@ proc createEth2Node*(rng: ref HmacDrbgContext,
       behaviourPenaltyWeight: -15.9,
       behaviourPenaltyDecay: 0.986,
       disconnectBadPeers: true,
-      directPeers:
-        block:
-          var res = initTable[PeerId, seq[MultiAddress]]()
-          if config.directPeers.len > 0:
-            for s in config.directPeers:
-              let
-                maddress = MultiAddress.init(s).tryGet()
-                mpeerId = maddress[multiCodec("p2p")].tryGet()
-                peerId = PeerId.init(mpeerId.protoAddress().tryGet()).tryGet()
-              res.mgetOrPut(peerId, @[]).add(maddress)
-              info "Adding priviledged direct peer", peerId, address = maddress
-          res
-        ,
+      directPeers: directPeers,
       bandwidthEstimatebps: config.bandwidthEstimate.get(100_000_000)
     )
     pubsub = GossipSub.init(
@@ -2402,7 +2414,7 @@ proc createEth2Node*(rng: ref HmacDrbgContext,
   let node = Eth2Node.new(
     config, cfg, enrForkId, discoveryForkId, forkDigests, getBeaconTime, switch, pubsub, extIp,
     extTcpPort, extUdpPort, netKeys.seckey.asEthKey,
-    discovery = config.discv5Enabled, rng = rng)
+    discovery = config.discv5Enabled, directPeers, rng = rng)
 
   node.pubsub.subscriptionValidator =
     proc(topic: string): bool {.gcsafe, raises: [].} =