go/runtime/host/sandbox: Retry Call in case the runtime is not yet ready

.changelog/2967.bugfix.md

@@ -0,0 +1 @@
+go/runtime/host/sandbox: Retry Call in case the runtime is not yet ready

go/runtime/host/sandbox/sandbox.go

@@ -98,15 +98,26 @@ func (r *sandboxedRuntime) ID() common.Namespace {
 }
 
 // Implements host.Runtime.
-func (r *sandboxedRuntime) Call(ctx context.Context, body *protocol.Body) (*protocol.Body, error) {
-	r.RLock()
-	conn := r.conn
-	r.RUnlock()
-
-	if conn == nil {
-		return nil, fmt.Errorf("runtime is not ready")
+func (r *sandboxedRuntime) Call(ctx context.Context, body *protocol.Body) (rsp *protocol.Body, err error) {
+	callFn := func() error {
+		r.RLock()
+		conn := r.conn
+		r.RUnlock()
+
+		if conn == nil {
+			return fmt.Errorf("runtime is not ready")
+		}
+		rsp, err = r.conn.Call(ctx, body)
+		if err != nil {
+			// All protocol-level errors are permanent.
+			return backoff.Permanent(err)
+		}
+		return nil
 	}
-	return r.conn.Call(ctx, body)
+
+	// Retry call in case the runtime is not yet ready.
+	err = backoff.Retry(callFn, backoff.WithContext(backoff.NewExponentialBackOff(), ctx))
+	return
 }
 
 // Implements host.Runtime.

go/worker/common/committee/runtime_host.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"errors"
 	"sync"
-	"time"
 
-	"github.com/cenkalti/backoff/v4"
 	"github.com/opentracing/opentracing-go"
 
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
@@ -20,11 +18,6 @@ import (
 	storage "github.com/oasisprotocol/oasis-core/go/storage/api"
 )
 
-const (
-	retryInterval = 1 * time.Second
-	maxRetries    = 15
-)
-
 var (
 	errMethodNotSupported   = errors.New("method not supported")
 	errEndpointNotSupported = errors.New("RPC endpoint not supported")
@@ -164,21 +157,7 @@ func (n *computeRuntimeHostNotifier) watchPolicyUpdates() {
 				SignedPolicyRaw: raw,
 			}}
 
-			var response *protocol.Body
-			call := func() error {
-				resp, err := n.host.Call(n.ctx, req)
-				if err != nil {
-					n.logger.Error("failed to dispatch RPC call to runtime",
-						"err", err,
-					)
-					return err
-				}
-				response = resp
-				return nil
-			}
-
-			retry := backoff.WithMaxRetries(backoff.NewConstantBackOff(retryInterval), maxRetries)
-			err := backoff.Retry(call, backoff.WithContext(retry, n.ctx))
+			response, err := n.host.Call(n.ctx, req)
 			if err != nil {
 				n.logger.Error("failed dispatching key manager policy update to runtime",
 					"err", err,