Merge commit '15967c407639ac48fcecf52f727555dae18a14b5' into dev/etan…

…/lc-dbprep

* commit '15967c407639ac48fcecf52f727555dae18a14b5':
  keep track of latest blocks for optimistic sync (#3715)
  separate non-zero exit code for doppelganger detection (#3728)
  cleanups: unused and duplicate imports, inconsistent naming conventions, URL updates (#3724)
  Change the log level for unrecognized json fields from debug to trace
  ncli: altair+ slot and state transitions (#3721)
  fix Nim 1.6 build deprecation warnings (#3712)
  remove unused stdlib imports (#3718)

AllTests-mainnet.md

@@ -55,6 +55,17 @@ OK: 5/5 Fail: 0/5 Skip: 0/5
 + basics                                                                                     OK
 ```
 OK: 1/1 Fail: 0/1 Skip: 0/1
+## Block clearance (light client) [Preset: mainnet]
+```diff
++ Delayed finality update                                                                    OK
++ Error conditions                                                                           OK
++ Incremental sync                                                                           OK
++ Initial sync                                                                               OK
++ Low slot numbers                                                                           OK
++ Reorg                                                                                      OK
++ Reverse incremental sync                                                                   OK
+```
+OK: 7/7 Fail: 0/7 Skip: 0/7
 ## Block pool altair processing [Preset: mainnet]
 ```diff
 + Invalid signatures [Preset: mainnet]                                                       OK
@@ -544,4 +555,4 @@ OK: 1/1 Fail: 0/1 Skip: 0/1
 OK: 1/1 Fail: 0/1 Skip: 0/1
 
 ---TOTAL---
-OK: 301/306 Fail: 0/306 Skip: 5/306
+OK: 308/313 Fail: 0/313 Skip: 5/313

Makefile

@@ -309,7 +309,9 @@ define CONNECT_TO_NETWORK_IN_DEV_MODE
 		--network=$(1) $(3) $(GOERLI_TESTNETS_PARAMS) \
 		--log-level="DEBUG; TRACE:discv5,networking; REQUIRED:none; DISABLED:none" \
 		--data-dir=build/data/shared_$(1)_$(NODE_ID) \
-		--serve-light-client-data=1 --import-light-client-data=only-new \
+		--light-client-enable=on \
+		--serve-light-client-data=on \
+		--import-light-client-data=only-new \
 		--dump $(NODE_PARAMS)
 endef
 

beacon_chain/beacon_chain_db_immutable.nim

@@ -8,9 +8,6 @@
 {.push raises: [Defect].}
 
 import
-  stew/results,
-  serialization,
-  eth/db/kvstore,
   ./spec/datatypes/[base, altair, bellatrix],
   ./spec/[eth2_ssz_serialization, eth2_merkleization]
 

beacon_chain/beacon_node.nim

@@ -22,17 +22,17 @@ import
     blockchain_dag, block_quarantine, exit_pool, attestation_pool,
     sync_committee_msg_pool],
   ./spec/datatypes/base,
-  ./sync/[sync_manager, request_manager],
+  ./sync/[optimistic_sync_light_client, sync_manager, request_manager],
   ./validators/[action_tracker, validator_monitor, validator_pool],
   ./rpc/state_ttl_cache
 
 export
   osproc, chronos, httpserver, presto, action_tracker,
   beacon_clock, beacon_chain_db, conf, light_client,
   attestation_pool, sync_committee_msg_pool, validator_pool,
-  eth2_network, eth1_monitor, request_manager, sync_manager,
-  eth2_processor, blockchain_dag, block_quarantine, base, exit_pool,
-  validator_monitor, consensus_manager
+  eth2_network, eth1_monitor, optimistic_sync_light_client,
+  request_manager, sync_manager, eth2_processor, blockchain_dag,
+  block_quarantine, base, exit_pool, validator_monitor, consensus_manager
 
 type
   RpcServer* = RpcHttpServer
@@ -45,6 +45,7 @@ type
     db*: BeaconChainDB
     config*: BeaconNodeConf
     attachedValidators*: ref ValidatorPool
+    lcOptSync*: LCOptimisticSync
     lightClient*: LightClient
     dag*: ChainDAGRef
     quarantine*: ref Quarantine

beacon_chain/beacon_node_light_client.nim

@@ -29,24 +29,71 @@ proc initLightClient*(
   # because the light client module also handles gossip subscriptions
   # for broadcasting light client data as a server.
 
-  let lightClient = createLightClient(
-    node.network, rng, config, cfg,
-    forkDigests, getBeaconTime, genesis_validators_root)
+  let
+    optimisticProcessor = proc(signedBlock: ForkedMsgTrustedSignedBeaconBlock):
+        Future[void] {.async.} =
+      debug "New LC optimistic block",
+        opt = signedBlock.toBlockId(),
+        dag = node.dag.head.bid,
+        wallSlot = node.currentSlot
+      return
+    optSync = initLCOptimisticSync(
+      node.network, getBeaconTime, optimisticProcessor,
+      config.safeSlotsToImportOptimistically)
+
+    lightClient = createLightClient(
+      node.network, rng, config, cfg,
+      forkDigests, getBeaconTime, genesis_validators_root)
 
   if config.lightClientEnable.get:
+    proc shouldSyncOptimistically(slot: Slot): bool =
+      const
+        # Minimum number of slots to be ahead of DAG to use optimistic sync
+        minProgress = 8 * SLOTS_PER_EPOCH
+        # Maximum age of light client optimistic header to use optimistic sync
+        maxAge = 2 * SLOTS_PER_EPOCH
+
+      if slot < getStateField(node.dag.headState, slot) + minProgress:
+        false
+      elif getBeaconTime().slotOrZero > slot + maxAge:
+        false
+      else:
+        true
+
+    proc onFinalizedHeader(lightClient: LightClient) =
+      let optimisticHeader = lightClient.optimisticHeader.valueOr:
+        return
+      if not shouldSyncOptimistically(optimisticHeader.slot):
+        return
+      let finalizedHeader = lightClient.finalizedHeader.valueOr:
+        return
+      optSync.setOptimisticHeader(optimisticHeader)
+      optSync.setFinalizedHeader(finalizedHeader)
+
+    proc onOptimisticHeader(lightClient: LightClient) =
+      let optimisticHeader = lightClient.optimisticHeader.valueOr:
+        return
+      if not shouldSyncOptimistically(optimisticHeader.slot):
+        return
+      optSync.setOptimisticHeader(optimisticHeader)
+
+    lightClient.onFinalizedHeader = onFinalizedHeader
+    lightClient.onOptimisticHeader = onOptimisticHeader
     lightClient.trustedBlockRoot = config.lightClientTrustedBlockRoot
 
   elif config.lightClientTrustedBlockRoot.isSome:
     warn "Ignoring `lightClientTrustedBlockRoot`, light client not enabled",
       lightClientEnable = config.lightClientEnable.get,
       lightClientTrustedBlockRoot = config.lightClientTrustedBlockRoot
 
+  node.lcOptSync = optSync
   node.lightClient = lightClient
 
 proc startLightClient*(node: BeaconNode) =
   if not node.config.lightClientEnable.get:
     return
 
+  node.lcOptSync.start()
   node.lightClient.start()
 
 proc installLightClientMessageValidators*(node: BeaconNode) =

beacon_chain/conf.nim

@@ -517,7 +517,7 @@ type
         desc: "Modify SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY"
         # https://github.com/ethereum/consensus-specs/blob/v1.2.0-rc.1/sync/optimistic.md#constants
         defaultValue: 128
-        name: "safe-slots-to-import-optimistically" }: uint64
+        name: "safe-slots-to-import-optimistically" }: uint16
 
       # Same option as appears in Lighthouse and Prysm
       # https://lighthouse-book.sigmaprime.io/suggested-fee-recipient.html

beacon_chain/consensus_object_pools/attestation_pool.nim

@@ -490,7 +490,7 @@ func init(
           if participation_bitmap[validator_index] != 0:
             # If any flag got set, there was an attestation from this validator.
             validator_bits[index_in_committee] = true
-        result.add((slot, committee_index.uint64), validator_bits)
+        result[(slot, committee_index.uint64)] = validator_bits
 
   # This treats all types of rewards as equivalent, which isn't ideal
   update_attestation_pool_cache(