diff --git a/.buildkite/scripts/common_e2e.sh b/.buildkite/scripts/common_e2e.sh index 6a485c05cbe..83efe34a951 100644 --- a/.buildkite/scripts/common_e2e.sh +++ b/.buildkite/scripts/common_e2e.sh @@ -158,12 +158,20 @@ run_backend_tendermint_committee() { rm -Rf {$ias_dir} if [ "${EKIDEN_TEE_HARDWARE}" == "intel-sgx" ]; then + # Note: This can just use a real client and watch the + # registry by setting `--address` and `--ias.wait_runtimes` + # to the appropriate values. + # + # nb: The startup order of things would need to be changed, + # and the brittle test cases will probably break in mysterious + # ways. if [ "${EKIDEN_UNSAFE_SKIP_AVR_VERIFY}" == "" ]; then # TODO: Ensure that IAS credentials are configured. ${EKIDEN_NODE} \ ias proxy \ --datadir ${ias_dir} \ - --ias.debug.skip_auth \ + --ias.use_genesis \ + --genesis.file ${genesis_file} \ --ias.auth.cert ${EKIDEN_IAS_CERT} \ --ias.auth.cert.ca ${EKIDEN_IAS_CERT} \ --ias.auth.cert.key ${EKIDEN_IAS_KEY} \ @@ -177,9 +185,9 @@ run_backend_tendermint_committee() { ${EKIDEN_NODE} \ ias proxy \ --datadir ${ias_dir} \ - --ias.debug.mock \ --ias.use_genesis \ --genesis.file ${genesis_file} \ + --ias.debug.mock \ --ias.spid 9b3085a55a5863f7cc66b380dcad0082 \ --debug.allow_test_keys \ --metrics.mode none \ @@ -189,13 +197,13 @@ run_backend_tendermint_committee() { fi EKIDEN_IAS_PROXY_ENABLED=1 + EKIDEN_IAS_PROXY_PORT=${ias_proxy_port} EKIDEN_IAS_PROXY_CERT=${ias_dir}/ias_proxy_cert.pem fi # Export some variables so compute workers can find them. EKIDEN_COMMITTEE_DIR=${committee_dir} EKIDEN_VALIDATOR_SOCKET=${base_datadir}-1/internal.sock - EKIDEN_IAS_PROXY_PORT=${ias_proxy_port} EKIDEN_GENESIS_FILE=${genesis_file} EKIDEN_EPOCHTIME_BACKEND=${epochtime_backend} EKIDEN_ENTITY_DESCRIPTOR=${entity_dir}/entity.json diff --git a/go/ekiden/cmd/ias/auth.go b/go/ekiden/cmd/ias/auth.go new file mode 100644 index 00000000000..37c50058969 --- /dev/null +++ b/go/ekiden/cmd/ias/auth.go @@ -0,0 +1,71 @@ +package ias + +import ( + "bytes" + "sync" + + "github.com/pkg/errors" + + "github.com/oasislabs/ekiden/go/common/cbor" + "github.com/oasislabs/ekiden/go/common/crypto/signature" + "github.com/oasislabs/ekiden/go/common/node" + "github.com/oasislabs/ekiden/go/common/sgx" + "github.com/oasislabs/ekiden/go/common/sgx/ias" + registry "github.com/oasislabs/ekiden/go/registry/api" +) + +type enclaveStore struct { + sync.RWMutex + + enclaves map[signature.MapKey][]sgx.EnclaveIdentity +} + +func (st *enclaveStore) verifyEvidence(evidence *ias.Evidence) error { + st.RLock() + defer st.RUnlock() + + enclaveIDs, ok := st.enclaves[evidence.ID.ToMapKey()] + if !ok { + return errors.New("ias: unknown runtime") + } + + quote, err := ias.DecodeQuote(evidence.Quote) + if err != nil { + return errors.Wrap(err, "ias: evidence contains an invalid quote") + } + + var id sgx.EnclaveIdentity + id.FromComponents(quote.Report.MRSIGNER, quote.Report.MRENCLAVE) + + for _, v := range enclaveIDs { + if bytes.Equal(v[:], id[:]) { + return nil + } + } + + return errors.New("ias: enclave identity not in runtime descriptor") +} + +func (st *enclaveStore) addRuntime(runtime *registry.Runtime) (int, error) { + st.Lock() + defer st.Unlock() + + if runtime.TEEHardware != node.TEEHardwareIntelSGX { + return len(st.enclaves), nil + } + + var vi registry.VersionInfoIntelSGX + if err := cbor.Unmarshal(runtime.Version.TEE, &vi); err != nil { + return len(st.enclaves), err + } + + st.enclaves[runtime.ID.ToMapKey()] = vi.Enclaves + + return len(st.enclaves), nil +} + +func newEnclaveStore() *enclaveStore { + return &enclaveStore{ + enclaves: make(map[signature.MapKey][]sgx.EnclaveIdentity), + } +} diff --git a/go/ekiden/cmd/ias/auth_genesis.go b/go/ekiden/cmd/ias/auth_genesis.go index 67bdae3e588..c1c7577682e 100644 --- a/go/ekiden/cmd/ias/auth_genesis.go +++ b/go/ekiden/cmd/ias/auth_genesis.go @@ -1,23 +1,17 @@ package ias import ( - "bytes" - - "github.com/pkg/errors" - - "github.com/oasislabs/ekiden/go/common/cbor" "github.com/oasislabs/ekiden/go/common/crypto/signature" "github.com/oasislabs/ekiden/go/common/logging" - "github.com/oasislabs/ekiden/go/common/node" - "github.com/oasislabs/ekiden/go/common/sgx" "github.com/oasislabs/ekiden/go/common/sgx/ias" "github.com/oasislabs/ekiden/go/genesis" registry "github.com/oasislabs/ekiden/go/registry/api" ) type genesisAuthenticator struct { - logger *logging.Logger - enclaves map[signature.MapKey][]sgx.EnclaveIdentity + logger *logging.Logger + + enclaves *enclaveStore } func (auth *genesisAuthenticator) VerifyEvidence(signer signature.PublicKey, evidence *ias.Evidence) error { @@ -25,40 +19,19 @@ func (auth *genesisAuthenticator) VerifyEvidence(signer signature.PublicKey, evi // validate that the signer is a node scheduled for the appropriate // runtime. - enclaveIDs, ok := auth.enclaves[evidence.ID.ToMapKey()] - if !ok { - auth.logger.Error("not a genesis runtime", - "id", evidence.ID, - ) - return errors.New("ias: not a runtime specified in genesis") - } - - quote, err := ias.DecodeQuote(evidence.Quote) + err := auth.enclaves.verifyEvidence(evidence) if err != nil { - auth.logger.Error("evidence contains an invalid quote", + auth.logger.Error("rejecting proxy request, invalid runtime", "err", err, + "id", evidence.ID, ) - return errors.Wrap(err, "ias: evidence contains an invalid quote") + return err } - var id sgx.EnclaveIdentity - id.FromComponents(quote.Report.MRSIGNER, quote.Report.MRENCLAVE) - - for _, v := range enclaveIDs { - if bytes.Equal(v[:], id[:]) { - auth.logger.Debug("found enclave identity in genesis runtime descriptor", - "id", evidence.ID, - "enclave_identity", id, - ) - return nil - } - } - - auth.logger.Error("enclave identity not in genesis runtime descriptor", + auth.logger.Debug("allowing proxy request, found enclave identity", "id", evidence.ID, - "enclave_identity", id, ) - return errors.New("ias: enclave identity not in genesis runtime descriptor") + return nil } func newGenesisAuthenticator() (ias.GRPCAuthenticator, error) { @@ -74,7 +47,7 @@ func newGenesisAuthenticator() (ias.GRPCAuthenticator, error) { auth := &genesisAuthenticator{ logger: logging.GetLogger("cmd/ias/proxy/auth/genesis"), - enclaves: make(map[signature.MapKey][]sgx.EnclaveIdentity), + enclaves: newEnclaveStore(), } for _, v := range doc.Registry.Runtimes { var rt registry.Runtime @@ -82,19 +55,9 @@ func newGenesisAuthenticator() (ias.GRPCAuthenticator, error) { return nil, err } - if rt.TEEHardware != node.TEEHardwareIntelSGX { - continue - } - if len(rt.Version.TEE) == 0 { - continue - } - - var vi registry.VersionInfoIntelSGX - if err = cbor.Unmarshal(rt.Version.TEE, &vi); err != nil { + if _, err = auth.enclaves.addRuntime(&rt); err != nil { return nil, err } - - auth.enclaves[rt.ID.ToMapKey()] = vi.Enclaves } return auth, nil diff --git a/go/ekiden/cmd/ias/auth_registry.go b/go/ekiden/cmd/ias/auth_registry.go new file mode 100644 index 00000000000..b0cb9509e11 --- /dev/null +++ b/go/ekiden/cmd/ias/auth_registry.go @@ -0,0 +1,134 @@ +package ias + +import ( + "context" + "io" + "sync" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "google.golang.org/grpc" + + "github.com/oasislabs/ekiden/go/common/crypto/signature" + "github.com/oasislabs/ekiden/go/common/logging" + "github.com/oasislabs/ekiden/go/common/sgx/ias" + cmdGrpc "github.com/oasislabs/ekiden/go/ekiden/cmd/common/grpc" + grpcRegistry "github.com/oasislabs/ekiden/go/grpc/registry" + registry "github.com/oasislabs/ekiden/go/registry/api" +) + +type registryAuthenticator struct { + logger *logging.Logger + + conn *grpc.ClientConn + client grpcRegistry.RuntimeRegistryClient + + enclaves *enclaveStore + + initOnce sync.Once + initCh chan struct{} +} + +func (auth *registryAuthenticator) VerifyEvidence(signer signature.PublicKey, evidence *ias.Evidence) error { + <-auth.initCh + + // TODO: This could/should do something clever with respect to verifying + // the signer, but node registration currently requires attestations for + // all of the runtimes that the node supports, leading to a chicken/egg + // situation. + // + // Revisit this after we reconsider the node registration process. + + err := auth.enclaves.verifyEvidence(evidence) + if err != nil { + auth.logger.Error("rejecting proxy request, invalid runtime", + "err", err, + "id", evidence.ID, + ) + return err + } + + auth.logger.Debug("allowing proxy request, found enclave identity", + "id", evidence.ID, + ) + return nil +} + +func (auth *registryAuthenticator) worker(ctx context.Context) { + defer auth.conn.Close() + + waitRuntimes := viper.GetInt(cfgWaitRuntimes) + if waitRuntimes <= 0 { + close(auth.initCh) + } + + stream, err := auth.client.WatchRuntimes(ctx, &grpcRegistry.WatchRuntimesRequest{}) + if err != nil { + auth.logger.Error("failed to start the WatchRuntimes stream", + "err", err, + ) + panic("unable to watch runtimes") + } + + for { + select { + case <-ctx.Done(): + return + default: + } + + pb, err := stream.Recv() + if err == io.EOF { + auth.logger.Error("data source stream closed by peer") + panic("data source disappeared") + } + if err != nil { + auth.logger.Error("runtime stream returned error", + "err", err, + ) + continue + } + + var runtime registry.Runtime + if err = runtime.FromProto(pb.GetRuntime()); err != nil { + auth.logger.Error("malformed runtime protobuf", + "err", err, + ) + continue + } + + n, err := auth.enclaves.addRuntime(&runtime) + if err != nil { + auth.logger.Error("failed to add runtime", + "err", err, + "id", runtime.ID, + ) + continue + } + if waitRuntimes > 0 && n == waitRuntimes { + auth.logger.Info("sufficient runtimes received, starting verification") + auth.initOnce.Do(func() { + close(auth.initCh) + }) + } + } +} + +func newRegistryAuthenticator(ctx context.Context, cmd *cobra.Command) (ias.GRPCAuthenticator, error) { + conn, err := cmdGrpc.NewClient(cmd) + if err != nil { + return nil, errors.Wrap(err, "ias: failed to create gRPC client") + } + + auth := ®istryAuthenticator{ + logger: logging.GetLogger("cmd/ias/proxy/auth/registry"), + conn: conn, + client: grpcRegistry.NewRuntimeRegistryClient(conn), + enclaves: newEnclaveStore(), + initCh: make(chan struct{}), + } + go auth.worker(ctx) + + return auth, nil +} diff --git a/go/ekiden/cmd/ias/proxy.go b/go/ekiden/cmd/ias/proxy.go index 9f26a7ff159..c339e298f08 100644 --- a/go/ekiden/cmd/ias/proxy.go +++ b/go/ekiden/cmd/ias/proxy.go @@ -2,6 +2,7 @@ package ias import ( + "context" "crypto/tls" "crypto/x509" "fmt" @@ -35,6 +36,7 @@ const ( cfgDebugMock = "ias.debug.mock" cfgDebugSkipAuth = "ias.debug.skip_auth" cfgUseGenesis = "ias.use_genesis" + cfgWaitRuntimes = "ias.wait_runtimes" tlsKeyFilename = "ias_proxy.pem" tlsCertFilename = "ias_proxy_cert.pem" @@ -153,7 +155,7 @@ func doProxy(cmd *cobra.Command, args []string) { } // Initialize the IAS proxy authenticator. - grpcAuth, err := grpcAuthenticatorFromFlags() + grpcAuth, err := grpcAuthenticatorFromFlags(env.svcMgr.Ctx, cmd) if err != nil { logger.Error("failed to initialize IAS gRPC authentiator", "err", err, @@ -234,7 +236,7 @@ func iasEndpointFromFlags() (ias.Endpoint, error) { return ias.NewIASEndpoint(cfg) } -func grpcAuthenticatorFromFlags() (ias.GRPCAuthenticator, error) { +func grpcAuthenticatorFromFlags(ctx context.Context, cmd *cobra.Command) (ias.GRPCAuthenticator, error) { if viper.GetBool(cfgDebugSkipAuth) { logger.Warn("IAS gRPC authentication disabled, proxy is open") return nil, nil @@ -243,7 +245,7 @@ func grpcAuthenticatorFromFlags() (ias.GRPCAuthenticator, error) { return newGenesisAuthenticator() } - return nil, fmt.Errorf("ias: no authentication configured") + return newRegistryAuthenticator(ctx, cmd) } // RegisterFlags registers the flags used by the proxy command. @@ -258,6 +260,7 @@ func RegisterFlags(cmd *cobra.Command) { cmd.Flags().Bool(cfgDebugMock, false, "generate mock IAS AVR responses (UNSAFE)") cmd.Flags().Bool(cfgDebugSkipAuth, false, "disable proxy authentication (UNSAFE)") cmd.Flags().Bool(cfgUseGenesis, false, "use a genesis document instead of the registry") + cmd.Flags().Int(cfgWaitRuntimes, 0, "wait for N runtimes to be registered before servicing requests") } for _, v := range []string{ @@ -270,6 +273,7 @@ func RegisterFlags(cmd *cobra.Command) { cfgDebugMock, cfgDebugSkipAuth, cfgUseGenesis, + cfgWaitRuntimes, } { _ = viper.BindPFlag(v, cmd.Flags().Lookup(v)) } @@ -282,6 +286,8 @@ func RegisterFlags(cmd *cobra.Command) { } { v(cmd) } + + cmdGrpc.RegisterClientFlags(cmd, false) } // Register registers the ias sub-command and all of it's children.