keymanager: Wait for keymanager to become ready

.changelog/2130.internal.md

@@ -0,0 +1,11 @@
+client protocol: Add IsReady() and WaitReady() RPC methods
+
+Beside `IsSynced()` and `WaitSynced()` which are triggered when a node is
+registered, new `IsReady()` and `WaitReady()` methods have been added to
+client protocol. These are triggered when all node workers have been
+initialized (including the runtimes) and the hosted processes are ready to
+process requests. Some oasis-test-runner scenarios have been updated
+accordingly.
+
+In addition new `oasis-node debug control wait-ready`
+command was added which blocks the client until the node is ready.

go/worker/keymanager/worker.go

@@ -184,29 +184,20 @@ func (w *Worker) callLocal(ctx context.Context, data []byte) ([]byte, error) {
 }
 
 func (w *Worker) updateStatus(status *api.Status, startedEvent *host.StartedEvent) error {
-	var initOk bool
 	defer func() {
-		if !initOk {
-			// TODO: once #2130 is done and keymanager reports as Ready only
-			// after initialization succeeds, change this to always pre-register
-			// on the initial updateStatus call (and not only after the first
-			// initialization fails as it is done currently).
-			// This is likely a new key manager that needs to replicate.
-			// Send a node registration anyway, so that other nodes know
-			// to update their access control.
-			w.roleProvider.SetAvailable(func(n *node.Node) error {
-				rt := n.AddOrUpdateRuntime(w.runtime.ID())
-				rt.Version = startedEvent.Version
-				rt.ExtraInfo = nil
-				rt.Capabilities.TEE = startedEvent.CapabilityTEE
-				return nil
-			})
-
-			// If initialization failed setup a retry ticker.
-			if w.initTicker == nil {
-				w.initTicker = backoff.NewTicker(backoff.NewExponentialBackOff())
-				w.initTickerCh = w.initTicker.C
-			}
+		<-w.Initialized()
+		w.roleProvider.SetAvailable(func(n *node.Node) error {
+			rt := n.AddOrUpdateRuntime(w.runtime.ID())
+			rt.Version = startedEvent.Version
+			rt.ExtraInfo = nil
+			rt.Capabilities.TEE = startedEvent.CapabilityTEE
+			return nil
+		})
+
+		// If initialization failed setup a retry ticker.
+		if w.initTicker == nil {
+			w.initTicker = backoff.NewTicker(backoff.NewExponentialBackOff())
+			w.initTickerCh = w.initTicker.C
 		}
 	}()
 
@@ -307,7 +298,6 @@ func (w *Worker) updateStatus(status *api.Status, startedEvent *host.StartedEven
 	}
 
 	// Register as we are now ready to handle requests.
-	initOk = true
 	w.roleProvider.SetAvailable(func(n *node.Node) error {
 		rt := n.AddOrUpdateRuntime(w.runtime.ID())
 		rt.Version = startedEvent.Version