go/keymanager: Add support for multiple nodes

[skip ci]

.buildkite/scripts/common_e2e.sh

@@ -158,6 +158,7 @@ run_backend_tendermint_committee() {
             --registry.backend tendermint \
             --roothash.backend tendermint \
             --roothash.tendermint.index_blocks \
+            --keymanager.backend tendermint \
             --genesis.file ${genesis_file} \
             --tendermint.core.listen_address tcp://0.0.0.0:${tm_port} \
             --tendermint.consensus.timeout_commit 250ms \
@@ -239,6 +240,7 @@ run_compute_node() {
         --scheduler.backend trivial \
         --registry.backend tendermint \
         --roothash.backend tendermint \
+        --keymanager.backend tendermint \
         --genesis.file ${EKIDEN_GENESIS_FILE} \
         --tendermint.core.listen_address tcp://0.0.0.0:${tm_port} \
         --tendermint.consensus.timeout_commit 250ms \
@@ -312,6 +314,7 @@ run_storage_node() {
         --scheduler.backend trivial \
         --registry.backend tendermint \
         --roothash.backend tendermint \
+        --keymanager.backend tendermint \
         --genesis.file ${EKIDEN_GENESIS_FILE} \
         --tendermint.core.listen_address tcp://0.0.0.0:${tm_port} \
         --tendermint.consensus.timeout_commit 250ms \
@@ -394,6 +397,7 @@ run_keymanager_node() {
         --scheduler.backend trivial \
         --registry.backend tendermint \
         --roothash.backend tendermint \
+        --keymanager.backend tendermint \
         --genesis.file ${EKIDEN_GENESIS_FILE} \
         --tendermint.core.listen_address tcp://0.0.0.0:${tm_port} \
         --tendermint.consensus.timeout_commit 250ms \
@@ -406,6 +410,7 @@ run_keymanager_node() {
         --worker.keymanager.runtime.loader ${EKIDEN_RUNTIME_LOADER} \
         --worker.keymanager.runtime.binary ${EKIDEN_ROOT_PATH}/target/${runtime_target}/debug/ekiden-keymanager-runtime${runtime_ext} \
         --worker.keymanager.runtime.id ${EKIDEN_KM_RUNTIME_ID} \
+        --worker.keymanager.may_generate \
         --tendermint.seeds "${EKIDEN_SEED_NODE_ID}@127.0.0.1:${EKIDEN_SEED_NODE_PORT}" \
         --datadir ${data_dir} \
         --debug.allow_test_keys \
@@ -449,6 +454,7 @@ run_seed_node() {
         --scheduler.backend trivial \
         --registry.backend tendermint \
         --roothash.backend tendermint \
+        --keymanager.backend tendermint \
         --tendermint.core.listen_address tcp://0.0.0.0:${EKIDEN_SEED_NODE_PORT} \
         --tendermint.seed_mode \
         --tendermint.debug.addr_book_lenient \

go/ekiden/cmd/node/node.go

@@ -29,6 +29,8 @@ import (
 	epochtimeAPI "github.com/oasislabs/ekiden/go/epochtime/api"
 	"github.com/oasislabs/ekiden/go/genesis"
 	"github.com/oasislabs/ekiden/go/ias"
+	"github.com/oasislabs/ekiden/go/keymanager"
+	keymanagerAPI "github.com/oasislabs/ekiden/go/keymanager/api"
 	keymanagerClient "github.com/oasislabs/ekiden/go/keymanager/client"
 	"github.com/oasislabs/ekiden/go/registry"
 	registryAPI "github.com/oasislabs/ekiden/go/registry/api"
@@ -45,7 +47,7 @@ import (
 	workerCommon "github.com/oasislabs/ekiden/go/worker/common"
 	"github.com/oasislabs/ekiden/go/worker/common/p2p"
 	"github.com/oasislabs/ekiden/go/worker/compute"
-	"github.com/oasislabs/ekiden/go/worker/keymanager"
+	keymanagerWorker "github.com/oasislabs/ekiden/go/worker/keymanager"
 	"github.com/oasislabs/ekiden/go/worker/merge"
 	"github.com/oasislabs/ekiden/go/worker/registration"
 	workerStorage "github.com/oasislabs/ekiden/go/worker/storage"
@@ -75,18 +77,20 @@ type Node struct {
 	grpcInternal *grpc.Server
 	svcTmnt      tmService.TendermintService
 
-	Genesis    genesis.Provider
-	Identity   *identity.Identity
-	Beacon     beaconAPI.Backend
-	Epochtime  epochtimeAPI.Backend
-	Registry   registryAPI.Backend
-	RootHash   roothashAPI.Backend
-	Scheduler  schedulerAPI.Backend
-	Staking    stakingAPI.Backend
-	Storage    storageAPI.Backend
-	IAS        *ias.IAS
-	Client     *client.Client
-	KeyManager *keymanagerClient.Client
+	Genesis   genesis.Provider
+	Identity  *identity.Identity
+	Beacon    beaconAPI.Backend
+	Epochtime epochtimeAPI.Backend
+	Registry  registryAPI.Backend
+	RootHash  roothashAPI.Backend
+	Scheduler schedulerAPI.Backend
+	Staking   stakingAPI.Backend
+	Storage   storageAPI.Backend
+	IAS       *ias.IAS
+	Client    *client.Client
+
+	KeyManager       keymanagerAPI.Backend
+	KeyManagerClient *keymanagerClient.Client
 
 	CommonWorker               *workerCommon.Worker
 	ComputeWorker              *compute.Worker
@@ -132,6 +136,9 @@ func (n *Node) initBackends() error {
 	if n.Staking, err = staking.New(n.svcMgr.Ctx, n.svcTmnt); err != nil {
 		return err
 	}
+	if n.KeyManager, err = keymanager.New(n.svcMgr.Ctx, n.Epochtime, n.Registry, n.svcTmnt); err != nil {
+		return err
+	}
 	n.svcMgr.RegisterCleanupOnly(n.Staking, "staking backend")
 	if n.Scheduler, err = scheduler.New(n.svcMgr.Ctx, n.Epochtime, n.Registry, n.Beacon, n.svcTmnt); err != nil {
 		return err
@@ -204,13 +211,14 @@ func (n *Node) initAndStartWorkers(logger *logging.Logger) error {
 	}
 	n.svcMgr.Register(n.WorkerRegistration)
 
-	// Initialize the key manager service.
-	kmSvc, kmEnabled, err := keymanager.New(
+	// Initialize the key manager worker service.
+	kmSvc, kmEnabled, err := keymanagerWorker.New(
 		dataDir,
 		n.IAS,
 		n.CommonWorker.Grpc,
 		n.WorkerRegistration,
 		&workerCommonCfg,
+		n.KeyManager,
 	)
 	if err != nil {
 		return err
@@ -247,7 +255,7 @@ func (n *Node) initAndStartWorkers(logger *logging.Logger) error {
 		n.CommonWorker,
 		n.MergeWorker,
 		n.IAS,
-		n.KeyManager,
+		n.KeyManagerClient,
 		n.WorkerRegistration,
 	)
 	if err != nil {
@@ -474,7 +482,7 @@ func NewNode() (*Node, error) {
 	logger.Info("starting ekiden node")
 
 	// Initialize the key manager client service.
-	node.KeyManager, err = keymanagerClient.New(node.Registry)
+	node.KeyManagerClient, err = keymanagerClient.New(node.KeyManager, node.Registry)
 	if err != nil {
 		logger.Error("failed to initialize key manager client",
 			"err", err,
@@ -491,7 +499,7 @@ func NewNode() (*Node, error) {
 		node.Scheduler,
 		node.Registry,
 		node.svcTmnt,
-		node.KeyManager,
+		node.KeyManagerClient,
 	)
 	if err != nil {
 		return nil, err
@@ -560,6 +568,7 @@ func RegisterFlags(cmd *cobra.Command) {
 		ias.RegisterFlags,
 		keymanager.RegisterFlags,
 		keymanagerClient.RegisterFlags,
+		keymanagerWorker.RegisterFlags,
 		client.RegisterFlags,
 		compute.RegisterFlags,
 		p2p.RegisterFlags,

go/ekiden/node_test.go

@@ -47,6 +47,7 @@ var (
 		{"log.level.default", "DEBUG"},
 		{"epochtime.backend", "tendermint_mock"},
 		{"beacon.backend", "tendermint"},
+		{"keymanager.backend", "tendermint"},
 		{"registry.backend", "tendermint"},
 		{"roothash.backend", "tendermint"},
 		{"roothash.tendermint.index_blocks", true},

go/keymanager/api/api.go

@@ -1,7 +1,65 @@
-// Package api implementes the key manager API and common data types.
+// Package api implementes the key manager management API and common data types.
 package api
 
-// InitResponse is the initialzation RPC response, returned as part of a
+import (
+	"context"
+	"errors"
+
+	"github.com/oasislabs/ekiden/go/common/cbor"
+	"github.com/oasislabs/ekiden/go/common/crypto/signature"
+	"github.com/oasislabs/ekiden/go/common/node"
+	"github.com/oasislabs/ekiden/go/common/pubsub"
+	registry "github.com/oasislabs/ekiden/go/registry/api"
+)
+
+var (
+	// ErrNoSuchKeyManager is the error returned when a key manager does not
+	// exist.
+	ErrNoSuchKeyManager = errors.New("keymanager: no such key manager")
+
+	// TestPublicKey is the insecure hardcoded key manager public key, used
+	// in insecure builds when a RAK is unavailable.
+	TestPublicKey signature.PublicKey
+
+	initResponseContext = []byte("EkKmIniR")
+)
+
+// Status is the current key manager status.
+type Status struct {
+	// ID is the runtime ID of the key manager.
+	ID signature.PublicKey `codec:"id"`
+
+	// IsInitialized is true iff the key manager is done initializing.
+	IsInitialized bool `codec:"is_initialized"`
+
+	// IsSecure is true iff the key manger is secure.
+	IsSecure bool `codec:"is_secure"`
+
+	// Checksum is the key manager master secret verification checksum.
+	Checksum []byte `codec:"checksum"`
+
+	// Nodes is the list of currently active key manager node IDs.
+	Nodes []signature.PublicKey `codec:"nodes"`
+
+	// TODO: Policy
+}
+
+// Backend is a key manager management implementation.
+type Backend interface {
+	// GetStatus returns a key manager status by key manager ID.
+	GetStatus(context.Context, signature.PublicKey) (*Status, error)
+
+	// GetStatuses returns all currently tracked key manager statuses.
+	GetStatuses(context.Context) ([]*Status, error)
+
+	// WatchStatuses returns a channel that produces a stream of messages
+	// containing the key manager statuses as it changes over time.
+	//
+	// Upon subscription the current status is sent immediately.
+	WatchStatuses() (<-chan *Status, *pubsub.Subscription)
+}
+
+// InitResponse is the initialization RPC response, returned as part of a
 // SignedInitResponse from the key manager enclave.
 type InitResponse struct {
 	IsSecure bool   `codec:"is_secure"`
@@ -14,3 +72,45 @@ type SignedInitResponse struct {
 	InitResponse InitResponse `codec:"init_response"`
 	Signature    []byte       `codec:"signature"`
 }
+
+func (r *SignedInitResponse) Verify(pk signature.PublicKey) error {
+	raw := cbor.Marshal(r.InitResponse)
+	if !pk.Verify(initResponseContext, raw, r.Signature) {
+		return errors.New("keymanager: invalid initialization response signature")
+	}
+	return nil
+}
+
+// VerifyExtraInfo verifies and parses the per-node + per-runtime ExtraInfo
+// blob for a key manager.
+func VerifyExtraInfo(rt *registry.Runtime, nodeRt *node.Runtime) (*InitResponse, error) {
+	var (
+		hw  node.TEEHardware
+		rak signature.PublicKey
+	)
+	if nodeRt.Capabilities.TEE == nil || nodeRt.Capabilities.TEE.Hardware == node.TEEHardwareInvalid {
+		hw = node.TEEHardwareInvalid
+		rak = TestPublicKey
+	} else {
+		// TODO: MRENCLAVE/MRSIGNER.
+		hw = nodeRt.Capabilities.TEE.Hardware
+		rak = nodeRt.Capabilities.TEE.RAK
+	}
+	if hw != rt.TEEHardware {
+		return nil, errors.New("keymanger: TEEHardware mismatch")
+	}
+
+	var untrustedSignedInitResponse SignedInitResponse
+	if err := cbor.Unmarshal(nodeRt.ExtraInfo, &untrustedSignedInitResponse); err != nil {
+		return nil, err
+	}
+	if err := untrustedSignedInitResponse.Verify(rak); err != nil {
+		return nil, err
+	}
+	return &untrustedSignedInitResponse.InitResponse, nil
+}
+
+func init() {
+	_ = TestPublicKey.UnmarshalHex("9d41a874b80e39a40c9644e964f0e4f967100c91654bfd7666435fe906af060f")
+	signature.RegisterTestPublicKey(TestPublicKey)
+}