more work on the BN/VC split

- fixed comments from the last review
- getting more data VIA RPC, moved some code back into the BN only
- attestation duties being requested as well

beacon_chain/beacon_node.nim

@@ -30,6 +30,7 @@ import
   validator_duties, validator_api
 
 const
+  genesisFile* = "genesis.ssz"
   hasPrompt = not defined(withoutPrompt)
 
 type
@@ -61,6 +62,58 @@ declareHistogram beacon_attestation_received_seconds_from_slot_start,
 
 logScope: topics = "beacnde"
 
+proc getStateFromSnapshot(conf: BeaconNodeConf): NilableBeaconStateRef =
+  var
+    genesisPath = conf.dataDir/genesisFile
+    snapshotContents: TaintedString
+    writeGenesisFile = false
+
+  if conf.stateSnapshot.isSome:
+    let
+      snapshotPath = conf.stateSnapshot.get.string
+      snapshotExt = splitFile(snapshotPath).ext
+
+    if cmpIgnoreCase(snapshotExt, ".ssz") != 0:
+      error "The supplied state snapshot must be a SSZ file",
+            suppliedPath = snapshotPath
+      quit 1
+
+    snapshotContents = readFile(snapshotPath)
+    if fileExists(genesisPath):
+      let genesisContents = readFile(genesisPath)
+      if snapshotContents != genesisContents:
+        error "Data directory not empty. Existing genesis state differs from supplied snapshot",
+              dataDir = conf.dataDir.string, snapshot = snapshotPath
+        quit 1
+    else:
+      debug "No previous genesis state. Importing snapshot",
+            genesisPath, dataDir = conf.dataDir.string
+      writeGenesisFile = true
+      genesisPath = snapshotPath
+  else:
+    try:
+      snapshotContents = readFile(genesisPath)
+    except CatchableError as err:
+      error "Failed to read genesis file", err = err.msg
+      quit 1
+
+  result = try:
+    newClone(SSZ.decode(snapshotContents, BeaconState))
+  except SerializationError:
+    error "Failed to import genesis file", path = genesisPath
+    quit 1
+
+  info "Loaded genesis state", path = genesisPath
+
+  if writeGenesisFile:
+    try:
+      notice "Writing genesis to data directory", path = conf.dataDir/genesisFile
+      writeFile(conf.dataDir/genesisFile, snapshotContents.string)
+    except CatchableError as err:
+      error "Failed to persist genesis file to data dir",
+        err = err.msg, genesisFile = conf.dataDir/genesisFile
+      quit 1
+
 proc enrForkIdFromState(state: BeaconState): ENRForkID =
   let
     forkVer = state.fork.current_version
@@ -557,7 +610,7 @@ proc installDebugApiHandlers(rpcServer: RpcServer, node: BeaconNode) =
 
 proc installRpcHandlers(rpcServer: RpcServer, node: BeaconNode) =
   # TODO: remove this if statement later - here just to test the config option for now
-  if node.config.externalValidators:
+  if node.config.validatorApi:
     rpcServer.installValidatorApiHandlers(node)
   rpcServer.installBeaconApiHandlers(node)
   rpcServer.installDebugApiHandlers(node)
@@ -823,9 +876,7 @@ when hasPrompt:
       # createThread(t, processPromptCommands, addr p)
 
 programMain:
-  let
-    banner = clientId & "\p" & copyrights & "\p\p" & nimBanner
-    config = BeaconNodeConf.load(version = banner, copyrightBanner = banner)
+  let config = makeBannerAndConfig(clientId, BeaconNodeConf)
 
   setupMainProc(config.logLevel)
 

beacon_chain/beacon_node_common.nim

@@ -135,3 +135,11 @@ proc updateHead*(node: BeaconNode): BlockRef =
   beacon_head_root.set newHead.root.toGaugeValue
 
   newHead
+
+template findIt*(s: openarray, predicate: untyped): int64 =
+  var res = -1
+  for i, it {.inject.} in s:
+    if predicate:
+      res = i
+      break
+  res

beacon_chain/conf.nim

@@ -111,10 +111,10 @@ type
         abbr: "v"
         name: "validator" }: seq[ValidatorKeyPath]
 
-      externalValidators* {.
+      validatorApi* {.
         defaultValue: false
-        desc: "Specify whether validators should be in an external process (a validator client) which communicates with the beacon node or they should be embedded."
-        name: "external-validators" }: bool
+        desc: "Specify whether the validator API should be enabled which would allow for external validators (validator clients) to use this beacon node."
+        name: "validator-api" }: bool
 
       stateSnapshot* {.
         desc: "Json file specifying a recent state snapshot."
@@ -319,11 +319,6 @@ type
         abbr: "v"
         name: "validator" }: seq[ValidatorKeyPath]
 
-      stateSnapshot* {.
-        desc: "Json file specifying a recent state snapshot."
-        abbr: "s"
-        name: "state-snapshot" }: Option[InputFile]
-
       delayStart* {.
         defaultValue: 0
         desc: "Seconds from now to delay the starting of the validator client (useful for debug purposes when starting before the beacon node in a script)."

beacon_chain/eth2_json_rpc_serialization.nim

@@ -3,7 +3,7 @@ import
   tables, json,
 
   # Nimble packages
-  stew/[bitseqs],
+  stew/[byteutils, bitseqs],
   json_rpc/jsonmarshal,
 
   # Local modules
@@ -31,6 +31,12 @@ proc fromJson*(n: JsonNode, argName: string, result: var ValidatorSig) =
 proc `%`*(value: ValidatorSig): JsonNode =
   result = newJString($value)
 
+proc fromJson*(n: JsonNode, argName: string, result: var Version) =
+  hexToByteArray(n.getStr(), array[4, byte](result))
+
+proc `%`*(value: Version): JsonNode =
+  result = newJString($value)
+
 template genFromJsonForIntType(t: untyped) =
   proc fromJson*(n: JsonNode, argName: string, result: var t) =
     n.kind.expect(JInt, argName)

beacon_chain/nimbus_binary_common.nim

@@ -5,75 +5,18 @@
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
 # at your option. This file may not be copied, modified, or distributed except according to those terms.
 
-# Common routines for a BeaconNode and a BeaconValidator node
+# Common routines for a BeaconNode and a ValidatorClient
 
 import
   # Standard library
-  os, tables, random, strutils,
+  tables, random, strutils,
 
   # Nimble packages
   chronos,
   chronicles, chronicles/helpers as chroniclesHelpers,
 
   # Local modules
-  spec/[datatypes, crypto],
-  conf,
-  block_pool, eth2_network
-
-const
-  genesisFile* = "genesis.ssz"
-
-proc getStateFromSnapshot*(conf: BeaconNodeConf|ValidatorClientConf): NilableBeaconStateRef =
-  var
-    genesisPath = conf.dataDir/genesisFile
-    snapshotContents: TaintedString
-    writeGenesisFile = false
-
-  if conf.stateSnapshot.isSome:
-    let
-      snapshotPath = conf.stateSnapshot.get.string
-      snapshotExt = splitFile(snapshotPath).ext
-
-    if cmpIgnoreCase(snapshotExt, ".ssz") != 0:
-      error "The supplied state snapshot must be a SSZ file",
-            suppliedPath = snapshotPath
-      quit 1
-
-    snapshotContents = readFile(snapshotPath)
-    if fileExists(genesisPath):
-      let genesisContents = readFile(genesisPath)
-      if snapshotContents != genesisContents:
-        error "Data directory not empty. Existing genesis state differs from supplied snapshot",
-              dataDir = conf.dataDir.string, snapshot = snapshotPath
-        quit 1
-    else:
-      debug "No previous genesis state. Importing snapshot",
-            genesisPath, dataDir = conf.dataDir.string
-      writeGenesisFile = true
-      genesisPath = snapshotPath
-  else:
-    try:
-      snapshotContents = readFile(genesisPath)
-    except CatchableError as err:
-      error "Failed to read genesis file", err = err.msg
-      quit 1
-
-  result = try:
-    newClone(SSZ.decode(snapshotContents, BeaconState))
-  except SerializationError:
-    error "Failed to import genesis file", path = genesisPath
-    quit 1
-
-  info "Loaded genesis state", path = genesisPath
-
-  if writeGenesisFile:
-    try:
-      notice "Writing genesis to data directory", path = conf.dataDir/genesisFile
-      writeFile(conf.dataDir/genesisFile, snapshotContents.string)
-    except CatchableError as err:
-      error "Failed to persist genesis file to data dir",
-        err = err.msg, genesisFile = conf.dataDir/genesisFile
-      quit 1
+  spec/[datatypes, crypto], eth2_network
 
 proc setupMainProc*(logLevel: string) =
   when compiles(defaultChroniclesStream.output.writer):
@@ -110,3 +53,7 @@ template ctrlCHandling*(extraCode: untyped) =
     info "Shutting down after having received SIGINT"
     extraCode
   setControlCHook(controlCHandler)
+
+template makeBannerAndConfig*(clientId: string, ConfType: type): untyped =
+  let banner = clientId & "\p" & copyrights & "\p\p" & nimBanner
+  ConfType.load(version = banner, copyrightBanner = banner)

beacon_chain/spec/eth2_apis/validator_callsigs.nim

@@ -8,10 +8,19 @@ import
 
 # TODO check which arguments are part of the path in the REST API
 
+
+
+# TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
+proc get_v1_beacon_states_fork(stateId: string): Fork
+
+# TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
+proc get_v1_beacon_genesis(): BeaconGenesisTuple
+
 proc get_v1_validator_blocks(slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig): BeaconBlock
 
 # TODO this doesn't have "validator" in it's path but is used by the validators nonetheless
-proc post_v1_beacon_blocks(body: SignedBeaconBlock)
+# TODO returns a bool even though in the API there is no return type - because of nim-json-rpc
+proc post_v1_beacon_blocks(body: SignedBeaconBlock): bool
 
 proc get_v1_validator_attestation_data(slot: Slot, committee_index: CommitteeIndex): AttestationData
 

beacon_chain/spec/eth2_apis/validator_callsigs_types.nim

@@ -15,7 +15,13 @@ type
     validator_committee_index*: uint64
     slot*: Slot
 
-  # TODO do we even need this? how about a simple tuple?
+  # TODO do we even need this? how about a simple tuple (alias)?
   ValidatorPubkeySlotPair* = object
     public_key*: ValidatorPubKey
     slot*: Slot
+
+  # TODO do we even need this? how about a simple tuple (alias)?
+  BeaconGenesisTuple* = object
+    genesis_time*: uint64
+    genesis_validators_root*: Eth2Digest
+    genesis_fork_version*: Version

beacon_chain/spec/validator.nim

@@ -220,7 +220,7 @@ func get_beacon_proposer_indexes_for_epoch*(state: BeaconState, epoch: Epoch, st
       result.add (currSlot, idx.get)
 
 # https://github.com/ethereum/eth2.0-specs/blob/v0.11.3/specs/phase0/validator.md#validator-assignments
-func get_committee_assignment(
+func get_committee_assignment*(
     state: BeaconState, epoch: Epoch, validator_index: ValidatorIndex):
     Option[tuple[a: seq[ValidatorIndex], b: CommitteeIndex, c: Slot]] {.used.} =
   # Return the committee assignment in the ``epoch`` for ``validator_index``.

beacon_chain/time.nim

@@ -27,19 +27,21 @@ type
 
   BeaconTime* = distinct int64 ## Seconds from beacon genesis time
 
-proc init*(T: type BeaconClock, state: BeaconState): T =
-  ## Initialize time from a beacon state. The genesis time of a beacon state is
-  ## constant throughout its lifetime, so the state from any slot will do,
-  ## including the genesis state.
-
+proc init*(T: type BeaconClock, genesis_time: uint64): T =
   let
-    unixGenesis = fromUnix(state.genesis_time.int64)
+    unixGenesis = fromUnix(genesis_time.int64)
     # GENESIS_SLOT offsets slot time, but to simplify calculations, we apply that
     # offset to genesis instead of applying it at every time conversion
     unixGenesisOffset = times.seconds(int(GENESIS_SLOT * SECONDS_PER_SLOT))
 
   T(genesis: unixGenesis - unixGenesisOffset)
 
+proc init*(T: type BeaconClock, state: BeaconState): T =
+  ## Initialize time from a beacon state. The genesis time of a beacon state is
+  ## constant throughout its lifetime, so the state from any slot will do,
+  ## including the genesis state.
+  BeaconClock.init(state.genesis_time)
+
 template `<`*(a, b: BeaconTime): bool =
   int64(a) < int64(b)
 

beacon_chain/validator_api.nim

@@ -26,8 +26,36 @@ type
 
 proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
 
+  # TODO Probably the `beacon` ones (and not `validator`) should be defined elsewhere...
+  rpcServer.rpc("get_v1_beacon_states_fork") do (stateId: string) -> Fork:
+    notice "== get_v1_beacon_states_fork", stateId = stateId
+    result = case stateId:
+      of "head":
+        discard node.updateHead() # TODO do we need this?
+        node.blockPool.headState.data.data.fork
+      of "genesis":
+        Fork(previous_version: Version(GENESIS_FORK_VERSION),
+             current_version: Version(GENESIS_FORK_VERSION),
+             epoch: 0.Epoch)
+      of "finalized":
+        # TODO
+        Fork()
+      of "justified":
+        # TODO
+        Fork()
+      else:
+        # TODO parse `stateId` as either a number (slot) or a hash (stateRoot)
+        Fork()
+
+  # TODO Probably the `beacon` ones (and not `validator`) should be defined elsewhere...
+  rpcServer.rpc("get_v1_beacon_genesis") do () -> BeaconGenesisTuple:
+    notice "== get_v1_beacon_genesis"
+    return BeaconGenesisTuple(genesis_time: node.blockPool.headState.data.data.genesis_time,
+                              genesis_validators_root: node.blockPool.headState.data.data.genesis_validators_root, 
+                              genesis_fork_version: Version(GENESIS_FORK_VERSION))
+
   rpcServer.rpc("get_v1_validator_blocks") do (slot: Slot, graffiti: Eth2Digest, randao_reveal: ValidatorSig) -> BeaconBlock:
-
+    notice "== get_v1_validator_blocks", slot = slot
     var head = node.updateHead()
 
     let proposer = node.blockPool.getProposer(head, slot)
@@ -38,11 +66,16 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
     let res = makeBeaconBlockForHeadAndSlot(node, valInfo, proposer.get()[0], graffiti, head, slot)
 
     # TODO how do we handle the case when we cannot return a meaningful block? 404...
-    doAssert(res.message.isSome())
-    return res.message.get()
-
-  rpcServer.rpc("post_v1_beacon_blocks") do (body: SignedBeaconBlock):
+    # currently this fails often - perhaps because the block has already been
+    # processed and signed with the inProcess validator...
+    # doAssert(res.message.isSome())
+    return res.message.get(BeaconBlock()) # returning a default if empty
+
+  rpcServer.rpc("post_v1_beacon_blocks") do (body: SignedBeaconBlock) -> bool :
+    notice "== post_v1_beacon_blocks"
+    # TODO make onBeaconBlock return a result and discard it wherever its unnecessary
     onBeaconBlock(node, body)
+    return true
 
   rpcServer.rpc("get_v1_validator_attestation_data") do (slot: Slot, committee_index: CommitteeIndex) -> AttestationData:
     discard
@@ -55,11 +88,22 @@ proc installValidatorApiHandlers*(rpcServer: RpcServer, node: BeaconNode) =
     discard
 
   rpcServer.rpc("post_v1_validator_duties_attester") do (epoch: Epoch, public_keys: seq[ValidatorPubKey]) -> seq[AttesterDuties]:
-    discard
+    notice "== post_v1_validator_duties_attester", epoch = epoch
+    for pubkey in public_keys:
+      let idx = node.blockPool.headState.data.data.validators.asSeq.findIt(it.pubKey == pubkey)
+      if idx != -1:
+        let res = node.blockPool.headState.data.data.get_committee_assignment(epoch, idx.ValidatorIndex)
+        if res.isSome:
+          result.add(AttesterDuties(public_key: pubkey,
+                                    committee_index: res.get.b,
+                                    committee_length: res.get.a.len.uint64,
+                                    validator_committee_index: res.get.a.find(idx.ValidatorIndex).uint64,
+                                    slot: res.get.c))
 
   rpcServer.rpc("get_v1_validator_duties_proposer") do (epoch: Epoch) -> seq[ValidatorPubkeySlotPair]:
+    notice "== get_v1_validator_duties_proposer", epoch = epoch
     var cache = get_empty_per_epoch_cache()
-    return get_beacon_proposer_indexes_for_epoch(node.blockPool.headState.data.data, epoch, cache).mapIt(ValidatorPubkeySlotPair(
+    result = get_beacon_proposer_indexes_for_epoch(node.blockPool.headState.data.data, epoch, cache).mapIt(ValidatorPubkeySlotPair(
         public_key: node.blockPool.headState.data.data.validators[it.i].pubkey,
         slot: it.s
       ))