Merge pull request #2843 from oasislabs/yawning/feature/km-replicatio…

…n-test

go/oasis-test-runner/oasis: Add a keymanager replication test

.changelog/2843.breaking.6.md

@@ -0,0 +1,8 @@
+keymanager-lib: Bind persisted state to the runtime ID
+
+It is likely prudent to bind the persisted master secret to the runtime
+ID.  This change does so by including the key manager runtime ID as the
+AAD when sealing the master secret.
+
+This is backward incompatible with all current key manager instances as
+the existing persisted master secret will not decrypt.

.changelog/2843.bugfix.1.md

@@ -0,0 +1 @@
+go/worker/keymanager: Add an enclave rpc handler

.changelog/2843.bugfix.2.md

@@ -0,0 +1 @@
+go/keymanager/client: Support km->km connections

.changelog/2843.bugfix.3.md

@@ -0,0 +1,4 @@
+go/worker/keymanager: Actually allow replication to maybe work
+
+Access control forbidding replication may be more secure, but is not all
+that useful.

.changelog/2843.feature.4.md

@@ -0,0 +1,3 @@
+go/keymanager/api: Add a gRPC endpoint for status queries
+
+Mostly so that the test cases can query statuses.

.changelog/2843.feature.5.md

@@ -0,0 +1 @@
+go/oasis-test-runner/oasis: Add a keymanager replication test

go/consensus/tendermint/keymanager/keymanager.go

@@ -11,14 +11,14 @@ import (
 	abcitypes "github.com/tendermint/tendermint/abci/types"
 	tmtypes "github.com/tendermint/tendermint/types"
 
-	"github.com/oasislabs/oasis-core/go/common"
 	"github.com/oasislabs/oasis-core/go/common/cbor"
 	"github.com/oasislabs/oasis-core/go/common/logging"
 	"github.com/oasislabs/oasis-core/go/common/pubsub"
 	consensus "github.com/oasislabs/oasis-core/go/consensus/api"
 	app "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/keymanager"
 	"github.com/oasislabs/oasis-core/go/consensus/tendermint/service"
 	"github.com/oasislabs/oasis-core/go/keymanager/api"
+	registry "github.com/oasislabs/oasis-core/go/registry/api"
 )
 
 type tendermintBackend struct {
@@ -30,13 +30,13 @@ type tendermintBackend struct {
 	notifier *pubsub.Broker
 }
 
-func (tb *tendermintBackend) GetStatus(ctx context.Context, id common.Namespace, height int64) (*api.Status, error) {
-	q, err := tb.querier.QueryAt(ctx, height)
+func (tb *tendermintBackend) GetStatus(ctx context.Context, query *registry.NamespaceQuery) (*api.Status, error) {
+	q, err := tb.querier.QueryAt(ctx, query.Height)
 	if err != nil {
 		return nil, err
 	}
 
-	return q.Status(ctx, id)
+	return q.Status(ctx, query.ID)
 }
 
 func (tb *tendermintBackend) GetStatuses(ctx context.Context, height int64) ([]*api.Status, error) {

go/keymanager/api/api.go

@@ -77,7 +77,7 @@ type Status struct {
 // Backend is a key manager management implementation.
 type Backend interface {
 	// GetStatus returns a key manager status by key manager ID.
-	GetStatus(context.Context, common.Namespace, int64) (*Status, error)
+	GetStatus(context.Context, *registry.NamespaceQuery) (*Status, error)
 
 	// GetStatuses returns all currently tracked key manager statuses.
 	GetStatuses(context.Context, int64) ([]*Status, error)
@@ -139,6 +139,9 @@ func VerifyExtraInfo(logger *logging.Logger, rt *registry.Runtime, nodeRt *node.
 	} else if err := registry.VerifyNodeRuntimeEnclaveIDs(logger, nodeRt, rt, ts); err != nil {
 		return nil, err
 	}
+	if nodeRt.ExtraInfo == nil {
+		return nil, fmt.Errorf("keymanager: missing ExtraInfo")
+	}
 
 	var untrustedSignedInitResponse SignedInitResponse
 	if err := cbor.Unmarshal(nodeRt.ExtraInfo, &untrustedSignedInitResponse); err != nil {

go/keymanager/api/grpc.go

@@ -4,14 +4,42 @@ import (
 	"context"
 	"fmt"
 
+	"google.golang.org/grpc"
+
 	"github.com/oasislabs/oasis-core/go/common/cbor"
+	cmnGrpc "github.com/oasislabs/oasis-core/go/common/grpc"
+	registry "github.com/oasislabs/oasis-core/go/registry/api"
 	enclaverpc "github.com/oasislabs/oasis-core/go/runtime/enclaverpc/api"
 )
 
 var (
 	// Make sure this always matches the appropriate method in
 	// `keymanager-runtime/src/methods.rs`.
 	getPublicKeyRequestMethod = "get_public_key"
+
+	// serviceName is the gRPC service name.
+	serviceName = cmnGrpc.NewServiceName("KeyManager")
+
+	// methodGetStatus is the GetStatus method.
+	methodGetStatus = serviceName.NewMethod("GetStatus", registry.NamespaceQuery{})
+	// methodGetStatuses is the GetStatuses method.
+	methodGetStatuses = serviceName.NewMethod("GetStatuses", int64(0))
+
+	// serviceDesc is the gRPC service descriptor.
+	serviceDesc = grpc.ServiceDesc{
+		ServiceName: string(serviceName),
+		HandlerType: (*Backend)(nil),
+		Methods: []grpc.MethodDesc{
+			{
+				MethodName: methodGetStatus.ShortName(),
+				Handler:    handlerGetStatus,
+			},
+			{
+				MethodName: methodGetStatuses.ShortName(),
+				Handler:    handlerGetStatuses,
+			},
+		},
+	}
 )
 
 type enclaveRPCEndpoint struct {
@@ -41,6 +69,83 @@ func (e *enclaveRPCEndpoint) AccessControlRequired(ctx context.Context, request
 	}
 }
 
+func handlerGetStatus( //nolint: golint
+	srv interface{},
+	ctx context.Context,
+	dec func(interface{}) error,
+	interceptor grpc.UnaryServerInterceptor,
+) (interface{}, error) {
+	var query registry.NamespaceQuery
+	if err := dec(&query); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(Backend).GetStatus(ctx, &query)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: methodGetStatus.FullName(),
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(Backend).GetStatus(ctx, req.(*registry.NamespaceQuery))
+	}
+	return interceptor(ctx, &query, info, handler)
+}
+
+func handlerGetStatuses( //nolint: golint
+	srv interface{},
+	ctx context.Context,
+	dec func(interface{}) error,
+	interceptor grpc.UnaryServerInterceptor,
+) (interface{}, error) {
+	var height int64
+	if err := dec(&height); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(Backend).GetStatuses(ctx, height)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: methodGetStatuses.FullName(),
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(Backend).GetStatuses(ctx, req.(int64))
+	}
+	return interceptor(ctx, height, info, handler)
+}
+
+// RegisterService registers a new keymanager backend service with the given gRPC server.
+func RegisterService(server *grpc.Server, service Backend) {
+	server.RegisterService(&serviceDesc, service)
+}
+
+// KeymanagerClient is a gRPC keymanager client.
+type KeymanagerClient struct {
+	conn *grpc.ClientConn
+}
+
+func (c *KeymanagerClient) GetStatus(ctx context.Context, query *registry.NamespaceQuery) (*Status, error) {
+	var resp Status
+	if err := c.conn.Invoke(ctx, methodGetStatus.FullName(), query, &resp); err != nil {
+		return nil, err
+	}
+	return &resp, nil
+}
+
+func (c *KeymanagerClient) GetStatuses(ctx context.Context, height int64) ([]*Status, error) {
+	var resp []*Status
+	if err := c.conn.Invoke(ctx, methodGetStatuses.FullName(), height, &resp); err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+// NewKeymanagerClient creates a new gRPC keymanager client service.
+func NewKeymanagerClient(c *grpc.ClientConn) *KeymanagerClient {
+	return &KeymanagerClient{c}
+}
+
 func init() {
 	enclaverpc.NewEndpoint(EnclaveRPCEndpoint, &enclaveRPCEndpoint{})
 }

go/keymanager/client/client.go

@@ -125,12 +125,21 @@ func (c *Client) worker() {
 		case rt := <-rtCh:
 			kmID = rt.KeyManager
 			if kmID == nil {
-				c.logger.Warn("runtime indicates no key manager is needed")
-				continue
+				if rt.Kind != registry.KindKeyManager {
+					c.logger.Warn("runtime indicates no key manager is needed")
+					continue
+				}
+
+				// We're a key manager client, that's interested in other
+				// instances of ourself.
+				kmID = &rt.ID
 			}
 
 			// Fetch current key manager status.
-			st, err := c.backend.GetStatus(c.ctx, *rt.KeyManager, consensus.HeightLatest)
+			st, err := c.backend.GetStatus(c.ctx, &registry.NamespaceQuery{
+				ID:     *kmID,
+				Height: consensus.HeightLatest,
+			})
 			if err != nil {
 				c.logger.Warn("failed to get key manager status",
 					"err", err,

go/oasis-net-runner/fixtures/default.go

@@ -111,6 +111,9 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) {
 		Validators: []oasis.ValidatorFixture{
 			oasis.ValidatorFixture{Entity: 1},
 		},
+		KeymanagerPolicies: []oasis.KeymanagerPolicyFixture{
+			oasis.KeymanagerPolicyFixture{Runtime: 0, Serial: 1},
+		},
 		Keymanagers: []oasis.KeymanagerFixture{
 			oasis.KeymanagerFixture{Runtime: 0, Entity: 1},
 		},

go/oasis-net-runner/fixtures/fixtures_test.go

@@ -20,6 +20,15 @@ func TestDefaultFixture(t *testing.T) {
 	require.Nil(t, err)
 	require.NotNil(t, data)
 
+	// As cool as having tests cases is, having to regenerate test data
+	// every single time the default fixture changes is incredibly
+	// annoying.
+	//
+	// May this pearl of wisdom serve as a guiding light for the next
+	// unfortunate victim.
+	//
+	// $ ./oasis-net-runner dump-fixture > /tmp/fuckfuckfuckfuckfuck
+
 	storedData, err := ioutil.ReadFile(defaultFixturePath)
 	require.Nil(t, err)
 	require.NotNil(t, storedData)

go/oasis-node/cmd/node/node.go

@@ -32,6 +32,7 @@ import (
 	genesisTestHelpers "github.com/oasislabs/oasis-core/go/genesis/tests"
 	"github.com/oasislabs/oasis-core/go/ias"
 	iasAPI "github.com/oasislabs/oasis-core/go/ias/api"
+	keymanagerAPI "github.com/oasislabs/oasis-core/go/keymanager/api"
 	cmdCommon "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/background"
 	"github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/flags"
@@ -185,6 +186,7 @@ func (n *Node) initBackends() error {
 	scheduler.RegisterService(grpcSrv, n.Consensus.Scheduler())
 	registryAPI.RegisterService(grpcSrv, n.Consensus.Registry())
 	stakingAPI.RegisterService(grpcSrv, n.Consensus.Staking())
+	keymanagerAPI.RegisterService(grpcSrv, n.Consensus.KeyManager())
 	consensusAPI.RegisterService(grpcSrv, n.Consensus)
 
 	cmdCommon.Logger().Debug("backends initialized")

go/oasis-test-runner/cmd/root.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime/debug"
 	"sort"
 	"strings"
 	"sync"
@@ -407,7 +408,7 @@ func runRoot(cmd *cobra.Command, args []string) error {
 func doScenario(childEnv *env.Env, sc scenario.Scenario) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
-			err = fmt.Errorf("root: panic caught running test case: %v", r)
+			err = fmt.Errorf("root: panic caught running test case: %v: %s", r, debug.Stack())
 		}
 	}()
 
@@ -470,7 +471,7 @@ func doScenario(childEnv *env.Env, sc scenario.Scenario) (err error) {
 func doCleanup(childEnv *env.Env) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
-			err = fmt.Errorf("root: panic caught cleaning up test case: %v", r)
+			err = fmt.Errorf("root: panic caught cleaning up test case: %v, %s", r, debug.Stack())
 		}
 	}()
 

go/oasis-test-runner/oasis/controller.go

@@ -6,6 +6,7 @@ import (
 	cmnGrpc "github.com/oasislabs/oasis-core/go/common/grpc"
 	consensus "github.com/oasislabs/oasis-core/go/consensus/api"
 	control "github.com/oasislabs/oasis-core/go/control/api"
+	keymanager "github.com/oasislabs/oasis-core/go/keymanager/api"
 	registry "github.com/oasislabs/oasis-core/go/registry/api"
 	runtimeClient "github.com/oasislabs/oasis-core/go/runtime/client/api"
 	staking "github.com/oasislabs/oasis-core/go/staking/api"
@@ -23,6 +24,7 @@ type Controller struct {
 	Registry      registry.Backend
 	RuntimeClient runtimeClient.RuntimeClient
 	Storage       storage.Backend
+	Keymanager    *keymanager.KeymanagerClient
 
 	conn *grpc.ClientConn
 }
@@ -52,6 +54,7 @@ func NewController(socketPath string) (*Controller, error) {
 		Registry:        registry.NewRegistryClient(conn),
 		RuntimeClient:   runtimeClient.NewRuntimeClient(conn),
 		Storage:         storage.NewStorageClient(conn),
+		Keymanager:      keymanager.NewKeymanagerClient(conn),
 
 		conn: conn,
 	}, nil