Add lightclient getHeadUpdate route (#3481)

* Add lightclient getHeadUpdate route

* Update sync test

packages/api/src/routes/lightclient.ts

@@ -1,8 +1,22 @@
 import {ContainerType, Path, VectorType} from "@chainsafe/ssz";
 import {Proof} from "@chainsafe/persistent-merkle-tree";
 import {altair, phase0, ssz, SyncPeriod} from "@chainsafe/lodestar-types";
-import {ArrayOf, ReturnTypes, RoutesData, Schema, sameType, ContainerData, ReqSerializers} from "../utils";
+import {
+  ArrayOf,
+  ReturnTypes,
+  RoutesData,
+  Schema,
+  sameType,
+  ContainerData,
+  ReqSerializers,
+  reqEmpty,
+  ReqEmpty,
+} from "../utils";
 import {queryParseProofPathsArr, querySerializeProofPathsArr} from "../utils/serdes";
+import {LightclientHeaderUpdate} from "./events";
+
+// Re-export for convenience when importing routes.lightclient.LightclientHeaderUpdate
+export {LightclientHeaderUpdate};
 
 // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes
 
@@ -27,6 +41,11 @@ export type Api = {
    * - Oldest update
    */
   getCommitteeUpdates(from: SyncPeriod, to: SyncPeriod): Promise<{data: altair.LightClientUpdate[]}>;
+  /**
+   * Returns the latest best head update available. Clients should use the SSE type `lightclient_header_update`
+   * unless to get the very first head update after syncing, or if SSE are not supported by the server.
+   */
+  getHeadUpdate(): Promise<{data: LightclientHeaderUpdate}>;
   /**
    * Fetch a snapshot with a proof to a trusted block root.
    * The trusted block root should be fetched with similar means to a weak subjectivity checkpoint.
@@ -41,12 +60,14 @@ export type Api = {
 export const routesData: RoutesData<Api> = {
   getStateProof: {url: "/eth/v1/lightclient/proof/:stateId", method: "GET"},
   getCommitteeUpdates: {url: "/eth/v1/lightclient/committee_updates", method: "GET"},
+  getHeadUpdate: {url: "/eth/v1/lightclient/head_update/", method: "GET"},
   getSnapshot: {url: "/eth/v1/lightclient/snapshot/:blockRoot", method: "GET"},
 };
 
 export type ReqTypes = {
   getStateProof: {params: {stateId: string}; query: {paths: string[]}};
   getCommitteeUpdates: {query: {from: number; to: number}};
+  getHeadUpdate: ReqEmpty;
   getSnapshot: {params: {blockRoot: string}};
 };
 
@@ -64,6 +85,8 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
       schema: {query: {from: Schema.UintRequired, to: Schema.UintRequired}},
     },
 
+    getHeadUpdate: reqEmpty,
+
     getSnapshot: {
       writeReq: (blockRoot) => ({params: {blockRoot}}),
       parseReq: ({params}) => [params.blockRoot],
@@ -87,10 +110,22 @@ export function getReturnTypes(): ReturnTypes<Api> {
     },
   });
 
+  const lightclientHeaderUpdate = new ContainerType<LightclientHeaderUpdate>({
+    fields: {
+      syncAggregate: ssz.altair.SyncAggregate,
+      header: ssz.phase0.BeaconBlockHeader,
+    },
+    casingMap: {
+      syncAggregate: "sync_aggregate",
+      header: "header",
+    },
+  });
+
   return {
     // Just sent the proof JSON as-is
     getStateProof: sameType(),
     getCommitteeUpdates: ContainerData(ArrayOf(ssz.altair.LightClientUpdate)),
+    getHeadUpdate: ContainerData(lightclientHeaderUpdate),
     getSnapshot: ContainerData(lightclientSnapshotWithProofType),
   };
 }

packages/api/test/unit/lightclient.test.ts

@@ -11,6 +11,8 @@ const root = Uint8Array.from(Buffer.alloc(32, 1));
 
 describe("lightclient", () => {
   const lightClientUpdate = ssz.altair.LightClientUpdate.defaultValue();
+  const syncAggregate = ssz.altair.SyncAggregate.defaultValue();
+  const header = ssz.phase0.BeaconBlockHeader.defaultValue();
 
   runGenericServerTest<Api, ReqTypes>(config, getClient, getRoutes, {
     getStateProof: {
@@ -41,11 +43,15 @@ describe("lightclient", () => {
       args: [1, 2],
       res: {data: [lightClientUpdate]},
     },
+    getHeadUpdate: {
+      args: [],
+      res: {data: {syncAggregate, header}},
+    },
     getSnapshot: {
       args: [toHexString(root)],
       res: {
         data: {
-          header: ssz.phase0.BeaconBlockHeader.defaultValue(),
+          header,
           currentSyncCommittee: lightClientUpdate.nextSyncCommittee,
           currentSyncCommitteeBranch: [root, root, root, root, root], // Vector(Root, 5)
         },

packages/light-client/src/index.ts

@@ -280,6 +280,15 @@ export class Lightclient {
           await new Promise((r) => setTimeout(r, ON_ERROR_RETRY_MS));
           continue;
         }
+
+        // Fetch latest head to prevent a potential 12 seconds lag between syncing and getting the first head,
+        // Don't retry, this is a non-critical UX improvement
+        try {
+          const {data: latestHeadUpdate} = await this.api.lightclient.getHeadUpdate();
+          this.processHeaderUpdate(latestHeadUpdate);
+        } catch (e) {
+          this.logger.error("Error fetching getHeadUpdate", {currentPeriod}, e as Error);
+        }
       }
 
       // After successfully syncing, track head if not already
@@ -289,6 +298,7 @@ export class Lightclient {
         this.logger.debug("Started tracking the head");
 
         // Subscribe to head updates over SSE
+        // TODO: Use polling for getHeadUpdate() is SSE is unavailable
         this.api.events.eventstream([routes.events.EventType.lightclientHeaderUpdate], controller.signal, this.onSSE);
       }
 

packages/light-client/test/lightclientApiServer.ts

@@ -42,6 +42,7 @@ export class LightclientServerApi implements routes.lightclient.Api {
   readonly states = new Map<RootHex, TreeBacked<allForks.BeaconState>>();
   readonly updates = new Map<SyncPeriod, altair.LightClientUpdate>();
   readonly snapshots = new Map<RootHex, routes.lightclient.LightclientSnapshotWithProof>();
+  latestHeadUpdate: routes.lightclient.LightclientHeaderUpdate | null = null;
 
   async getStateProof(stateId: string, paths: Path[]): Promise<{data: Proof}> {
     const state = this.states.get(stateId);
@@ -60,6 +61,11 @@ export class LightclientServerApi implements routes.lightclient.Api {
     return {data: updates};
   }
 
+  async getHeadUpdate(): Promise<{data: routes.lightclient.LightclientHeaderUpdate}> {
+    if (!this.latestHeadUpdate) throw Error("No latest head update");
+    return {data: this.latestHeadUpdate};
+  }
+
   async getSnapshot(blockRoot: string): Promise<{data: routes.lightclient.LightclientSnapshotWithProof}> {
     const snapshot = this.snapshots.get(blockRoot);
     if (!snapshot) throw Error(`snapshot for blockRoot ${blockRoot} not available`);

packages/light-client/test/unit/sync.test.ts

@@ -5,7 +5,14 @@ import {chainConfig as chainConfigDef} from "@chainsafe/lodestar-config/default"
 import {createIBeaconConfig, IChainConfig} from "@chainsafe/lodestar-config";
 import {Lightclient, LightclientEvent} from "../../src";
 import {EventsServerApi, LightclientServerApi, ServerOpts, startServer} from "../lightclientApiServer";
-import {computeLightclientUpdate, computeLightClientSnapshot, getInteropSyncCommittee, testLogger} from "../utils";
+import {
+  computeLightclientUpdate,
+  computeLightClientSnapshot,
+  getInteropSyncCommittee,
+  testLogger,
+  committeeUpdateToHeadUpdate,
+  lastInMap,
+} from "../utils";
 import {toHexString, TreeBacked} from "@chainsafe/ssz";
 import {expect} from "chai";
 
@@ -52,9 +59,13 @@ describe("Lightclient sync", () => {
 
     // Populate sync committee updates
     for (let period = initialPeriod; period <= targetPeriod; period++) {
-      lightclientServerApi.updates.set(period, computeLightclientUpdate(config, period));
+      const committeeUpdate = computeLightclientUpdate(config, period);
+      lightclientServerApi.updates.set(period, committeeUpdate);
     }
 
+    // So the first call to getHeadUpdate() doesn't error, store the latest snapshot as latest header update
+    lightclientServerApi.latestHeadUpdate = committeeUpdateToHeadUpdate(lastInMap(lightclientServerApi.updates));
+
     // Initilize from snapshot
     const lightclient = await Lightclient.initializeFromCheckpointRoot({
       config,
@@ -114,10 +125,13 @@ describe("Lightclient sync", () => {
           bodyRoot: SOME_HASH,
         };
 
-        eventsServerApi.emit({
-          type: routes.events.EventType.lightclientHeaderUpdate,
-          message: {header, syncAggregate: syncCommittee.signHeader(config, header)},
-        });
+        const headUpdate: routes.lightclient.LightclientHeaderUpdate = {
+          header,
+          syncAggregate: syncCommittee.signHeader(config, header),
+        };
+
+        lightclientServerApi.latestHeadUpdate = headUpdate;
+        eventsServerApi.emit({type: routes.events.EventType.lightclientHeaderUpdate, message: headUpdate});
       }
     });
 

packages/light-client/test/utils.ts

@@ -250,3 +250,21 @@ export function computeMerkleBranch(
   }
   return {root: value, proof};
 }
+
+export function committeeUpdateToHeadUpdate(
+  committeeUpdate: altair.LightClientUpdate
+): routes.lightclient.LightclientHeaderUpdate {
+  return {
+    header: committeeUpdate.finalityHeader,
+    syncAggregate: {
+      syncCommitteeBits: committeeUpdate.syncCommitteeBits,
+      syncCommitteeSignature: committeeUpdate.syncCommitteeSignature,
+    },
+  };
+}
+
+export function lastInMap<T>(map: Map<unknown, T>): T {
+  if (map.size === 0) throw Error("Empty map");
+  const values = Array.from(map.values());
+  return values[values.length - 1];
+}

packages/lodestar/src/api/impl/lightclient/index.ts

@@ -56,14 +56,16 @@ export function getLightclientApi(
 
     async getCommitteeUpdates(from, to) {
       const periods = linspace(from, to);
-      const updates = await Promise.all(
-        periods.map((period) => chain.lightClientServer.serveBestUpdateInPeriod(period))
-      );
+      const updates = await Promise.all(periods.map((period) => chain.lightClientServer.getCommitteeUpdates(period)));
       return {data: updates};
     },
 
+    async getHeadUpdate() {
+      return {data: await chain.lightClientServer.getHeadUpdate()};
+    },
+
     async getSnapshot(blockRoot) {
-      const snapshotProof = await chain.lightClientServer.serveInitCommittees(fromHexString(blockRoot));
+      const snapshotProof = await chain.lightClientServer.getSnapshot(fromHexString(blockRoot));
       return {data: snapshotProof};
     },
   };

packages/lodestar/src/chain/emitter.ts

@@ -1,10 +1,10 @@
 import {EventEmitter} from "events";
 import StrictEventEmitter from "strict-event-emitter-types";
 
+import {routes} from "@chainsafe/lodestar-api";
 import {phase0, Epoch, Slot, allForks} from "@chainsafe/lodestar-types";
 import {CheckpointWithHex, IProtoBlock} from "@chainsafe/lodestar-fork-choice";
 import {CachedBeaconState} from "@chainsafe/lodestar-beacon-state-transition";
-import {LightClientHeaderUpdate} from "./lightClient/types";
 import {AttestationError, BlockError} from "./errors";
 
 /**
@@ -123,7 +123,7 @@ export interface IChainEvents {
   [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithHex) => void;
   [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithHex) => void;
 
-  [ChainEvent.lightclientHeaderUpdate]: (headerUpdate: LightClientHeaderUpdate) => void;
+  [ChainEvent.lightclientHeaderUpdate]: (headerUpdate: routes.events.LightclientHeaderUpdate) => void;
 }
 
 /**

packages/lodestar/src/chain/lightClient/index.ts

@@ -169,6 +169,7 @@ export class LightClientServer {
    */
   private readonly prevHeadData = new Map<BlockRooHex, SyncAttestedData>();
   private checkpointHeaders = new Map<BlockRooHex, phase0.BeaconBlockHeader>();
+  private latestHeadUpdate: routes.lightclient.LightclientHeaderUpdate | null = null;
 
   private readonly zero: Pick<altair.LightClientUpdate, "finalityBranch" | "finalityHeader">;
 
@@ -218,7 +219,7 @@ export class LightClientServer {
   /**
    * API ROUTE to get `currentSyncCommittee` and `nextSyncCommittee` from a trusted state root
    */
-  async serveInitCommittees(blockRoot: Uint8Array): Promise<routes.lightclient.LightclientSnapshotWithProof> {
+  async getSnapshot(blockRoot: Uint8Array): Promise<routes.lightclient.LightclientSnapshotWithProof> {
     const syncCommitteeWitness = await this.db.syncCommitteeWitness.get(blockRoot);
     if (!syncCommitteeWitness) {
       throw Error(`syncCommitteeWitness not available ${toHexString(blockRoot)}`);
@@ -254,7 +255,7 @@ export class LightClientServer {
    * - Has the most bits
    * - Signed header at the oldest slot
    */
-  async serveBestUpdateInPeriod(period: SyncPeriod): Promise<altair.LightClientUpdate> {
+  async getCommitteeUpdates(period: SyncPeriod): Promise<altair.LightClientUpdate> {
     // Signature data
     const partialUpdate = await this.db.bestPartialLightClientUpdate.get(period);
     if (!partialUpdate) {
@@ -300,6 +301,17 @@ export class LightClientServer {
     }
   }
 
+  /**
+   * API ROUTE to poll LightclientHeaderUpdate.
+   * Clients should use the SSE type `lightclient_header_update` if available
+   */
+  async getHeadUpdate(): Promise<routes.lightclient.LightclientHeaderUpdate> {
+    if (this.latestHeadUpdate === null) {
+      throw Error("No latest header update available");
+    }
+    return this.latestHeadUpdate;
+  }
+
   /**
    * With forkchoice data compute which block roots will never become checkpoints and prune them.
    */
@@ -424,14 +436,18 @@ export class LightClientServer {
       }
     }
 
+    const headerUpdate: routes.lightclient.LightclientHeaderUpdate = {header: attestedData.header, syncAggregate};
+
     // Emit update
     // - At the earliest: 6 second after the slot start
     // - After a new update has INCREMENT_THRESHOLD == 32 bits more than the previous emitted threshold
-    this.emitter.emit(ChainEvent.lightclientHeaderUpdate, {
-      header: attestedData.header,
-      blockRoot: toHexString(attestedData.blockRoot),
-      syncAggregate,
-    });
+    this.emitter.emit(ChainEvent.lightclientHeaderUpdate, headerUpdate);
+
+    // Persist latest best update for getHeadUpdate()
+    // TODO: Once SyncAggregate are constructed from P2P too, count bits to decide "best"
+    if (!this.latestHeadUpdate || attestedData.header.slot > this.latestHeadUpdate.header.slot) {
+      this.latestHeadUpdate = headerUpdate;
+    }
 
     // Check if this update is better, otherwise ignore
     await this.maybeStoreNewBestPartialUpdate(syncAggregate, attestedData);

packages/lodestar/src/chain/lightClient/types.ts

@@ -1,4 +1,4 @@
-import {altair, phase0, RootHex} from "@chainsafe/lodestar-types";
+import {altair, phase0} from "@chainsafe/lodestar-types";
 
 /**
  * We aren't creating the sync committee proofs separately because our ssz library automatically adds leaves to composite types,
@@ -34,13 +34,6 @@ export type SyncCommitteeWitness = {
   nextSyncCommitteeRoot: Uint8Array;
 };
 
-export type LightClientHeaderUpdate = {
-  syncAggregate: altair.SyncAggregate;
-  header: phase0.BeaconBlockHeader;
-  /** Precomputed root to prevent re-hashing */
-  blockRoot: RootHex;
-};
-
 export type PartialLightClientUpdateFinalized = {
   isFinalized: true;
   header: phase0.BeaconBlockHeader;