From de723ec27b7ed94a9b80ae86e7ea0abbc720d256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrej=20Buko=C5=A1ek?= Date: Fri, 17 Apr 2020 11:48:15 +0200 Subject: [PATCH] go/oasis-test-runner: Test dynamic KM runtime registration --- .changelog/2840.trivial.md | 0 .../tendermint/apps/keymanager/keymanager.go | 3 +- .../apps/keymanager/transactions.go | 10 +- .../tendermint/keymanager/keymanager.go | 19 +- go/keymanager/api/policy_sgx.go | 2 +- go/oasis-node/cmd/keymanager/keymanager.go | 180 +++++++++--------- go/oasis-test-runner/oasis/cli/cli.go | 10 +- go/oasis-test-runner/oasis/cli/keymanager.go | 113 +++++++++++ go/oasis-test-runner/oasis/keymanager.go | 8 + go/oasis-test-runner/oasis/runtime.go | 16 ++ .../scenario/e2e/runtime_dynamic.go | 112 +++++++++-- 11 files changed, 360 insertions(+), 113 deletions(-) create mode 100644 .changelog/2840.trivial.md create mode 100644 go/oasis-test-runner/oasis/cli/keymanager.go diff --git a/.changelog/2840.trivial.md b/.changelog/2840.trivial.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/go/consensus/tendermint/apps/keymanager/keymanager.go b/go/consensus/tendermint/apps/keymanager/keymanager.go index 404cc695f75..31966c05e80 100644 --- a/go/consensus/tendermint/apps/keymanager/keymanager.go +++ b/go/consensus/tendermint/apps/keymanager/keymanager.go @@ -19,6 +19,7 @@ import ( stakingState "github.com/oasislabs/oasis-core/go/consensus/tendermint/apps/staking/state" epochtime "github.com/oasislabs/oasis-core/go/epochtime/api" "github.com/oasislabs/oasis-core/go/keymanager/api" + keymanager "github.com/oasislabs/oasis-core/go/keymanager/api" registry "github.com/oasislabs/oasis-core/go/registry/api" ) @@ -37,7 +38,7 @@ func (app *keymanagerApplication) ID() uint8 { } func (app *keymanagerApplication) Methods() []transaction.MethodName { - return nil + return keymanager.Methods } func (app *keymanagerApplication) Blessed() bool { diff --git a/go/consensus/tendermint/apps/keymanager/transactions.go b/go/consensus/tendermint/apps/keymanager/transactions.go index 13111026e72..ed1869d8844 100644 --- a/go/consensus/tendermint/apps/keymanager/transactions.go +++ b/go/consensus/tendermint/apps/keymanager/transactions.go @@ -33,7 +33,14 @@ func (app *keymanagerApplication) updatePolicy( // Get the existing policy document, if one exists. oldStatus, err := state.Status(ctx, rt.ID) - if err != nil { + switch err { + case nil: + case api.ErrNoSuchStatus: + // This must be a new key manager runtime. + oldStatus = &api.Status{ + ID: rt.ID, + } + default: return err } @@ -68,6 +75,7 @@ func (app *keymanagerApplication) updatePolicy( // will get updated. nodes, _ := regState.Nodes(ctx) registry.SortNodeList(nodes) + oldStatus.Policy = sigPol newStatus := app.generateStatus(ctx, rt, oldStatus, nodes) if err := state.SetStatus(ctx, newStatus); err != nil { panic(fmt.Errorf("failed to set keymanager status: %w", err)) diff --git a/go/consensus/tendermint/keymanager/keymanager.go b/go/consensus/tendermint/keymanager/keymanager.go index 8a67aec33a2..8bc930e5dab 100644 --- a/go/consensus/tendermint/keymanager/keymanager.go +++ b/go/consensus/tendermint/keymanager/keymanager.go @@ -8,6 +8,7 @@ import ( "github.com/eapache/channels" "github.com/pkg/errors" + abcitypes "github.com/tendermint/tendermint/abci/types" tmtypes "github.com/tendermint/tendermint/types" "github.com/oasislabs/oasis-core/go/common" @@ -88,17 +89,27 @@ func (tb *tendermintBackend) worker(ctx context.Context) { switch ev := event.(type) { case tmtypes.EventDataNewBlock: - tb.onEventDataNewBlock(ev) + tb.onEventDataNewBlock(ctx, ev) + case tmtypes.EventDataTx: + tb.onEventDataTx(ctx, ev) default: } } } -func (tb *tendermintBackend) onEventDataNewBlock(ev tmtypes.EventDataNewBlock) { - events := ev.ResultBeginBlock.GetEvents() +func (tb *tendermintBackend) onEventDataNewBlock(ctx context.Context, ev tmtypes.EventDataNewBlock) { + events := append([]abcitypes.Event{}, ev.ResultBeginBlock.GetEvents()...) events = append(events, ev.ResultEndBlock.GetEvents()...) - for _, tmEv := range events { + tb.onABCIEvents(ctx, events) +} + +func (tb *tendermintBackend) onEventDataTx(ctx context.Context, tx tmtypes.EventDataTx) { + tb.onABCIEvents(ctx, tx.Result.Events) +} + +func (tb *tendermintBackend) onABCIEvents(ctx context.Context, tmEvents []abcitypes.Event) { + for _, tmEv := range tmEvents { if tmEv.GetType() != app.EventType { continue } diff --git a/go/keymanager/api/policy_sgx.go b/go/keymanager/api/policy_sgx.go index f973b967c53..f4d80caa571 100644 --- a/go/keymanager/api/policy_sgx.go +++ b/go/keymanager/api/policy_sgx.go @@ -32,7 +32,7 @@ type EnclavePolicySGX struct { // // TODO: This could be made more sophisticated and seggregate based on // contract ID as well, but for now punt on the added complexity. - MayQuery map[signature.PublicKey][]sgx.EnclaveIdentity `json:"may_query"` + MayQuery map[common.Namespace][]sgx.EnclaveIdentity `json:"may_query"` // MayReplicate is the vector of enclave IDs that may retrieve the master // secret (Note: Each enclave ID may always implicitly replicate from other diff --git a/go/oasis-node/cmd/keymanager/keymanager.go b/go/oasis-node/cmd/keymanager/keymanager.go index baf2ac4a186..30dc7753ee7 100644 --- a/go/oasis-node/cmd/keymanager/keymanager.go +++ b/go/oasis-node/cmd/keymanager/keymanager.go @@ -28,22 +28,22 @@ import ( ) const ( - cfgPolicySerial = "keymanager.policy.serial" - cfgPolicyID = "keymanager.policy.id" - cfgPolicyFile = "keymanager.policy.file" - cfgPolicyEnclaveID = "keymanager.policy.enclave.id" - cfgPolicyMayQuery = "keymanager.policy.may.query" - cfgPolicyMayReplicate = "keymanager.policy.may.replicate" - cfgPolicyKeyFile = "keymanager.policy.key.file" - cfgPolicyTestKey = "keymanager.policy.testkey" - cfgPolicySigFile = "keymanager.policy.signature.file" - cfgPolicyIgnoreSig = "keymanager.policy.ignore.signature" - - cfgStatusFile = "keymanager.status.file" - cfgStatusID = "keymanager.status.id" - cfgStatusInitialized = "keymanager.status.initialized" - cfgStatusSecure = "keymanager.status.secure" - cfgStatusChecksum = "keymanager.status.checksum" + CfgPolicySerial = "keymanager.policy.serial" + CfgPolicyID = "keymanager.policy.id" + CfgPolicyFile = "keymanager.policy.file" + CfgPolicyEnclaveID = "keymanager.policy.enclave.id" + CfgPolicyMayQuery = "keymanager.policy.may.query" + CfgPolicyMayReplicate = "keymanager.policy.may.replicate" + CfgPolicyKeyFile = "keymanager.policy.key.file" + CfgPolicyTestKey = "keymanager.policy.testkey" + CfgPolicySigFile = "keymanager.policy.signature.file" + CfgPolicyIgnoreSig = "keymanager.policy.ignore.signature" + + CfgStatusFile = "keymanager.status.file" + CfgStatusID = "keymanager.status.id" + CfgStatusInitialized = "keymanager.status.initialized" + CfgStatusSecure = "keymanager.status.secure" + CfgStatusChecksum = "keymanager.status.checksum" policyFilename = "km_policy.cbor" statusFilename = "km_status.json" @@ -102,10 +102,10 @@ func doInitPolicy(cmd *cobra.Command, args []string) { } c := cbor.Marshal(p) - if err = ioutil.WriteFile(viper.GetString(cfgPolicyFile), c, 0666); err != nil { + if err = ioutil.WriteFile(viper.GetString(CfgPolicyFile), c, 0666); err != nil { logger.Error("failed to write key manager policy cbor file", "err", err, - "cfgPolicyFile", viper.GetString(cfgPolicyFile), + "CfgPolicyFile", viper.GetString(CfgPolicyFile), ) os.Exit(1) } @@ -117,15 +117,15 @@ func doInitPolicy(cmd *cobra.Command, args []string) { func policyFromFlags() (*kmApi.PolicySGX, error) { var id common.Namespace - if err := id.UnmarshalHex(viper.GetString(cfgPolicyID)); err != nil { + if err := id.UnmarshalHex(viper.GetString(CfgPolicyID)); err != nil { logger.Error("failed to parse key manager runtime ID", "err", err, - "cfgPolicyID", viper.GetString(cfgPolicyID), + "CfgPolicyID", viper.GetString(CfgPolicyID), ) return nil, err } - serial := viper.GetUint32(cfgPolicySerial) + serial := viper.GetUint32(CfgPolicySerial) enclaves := make(map[sgx.EnclaveIdentity]*kmApi.EnclavePolicySGX) @@ -133,7 +133,7 @@ func policyFromFlags() (*kmApi.PolicySGX, error) { // Since viper doesn't store order of arguments, go through os.Args by hand, // find --keymanager.policy.enclave.id and construct its permissions. for curArgIdx, curArg := range os.Args { - if curArg == "--"+cfgPolicyEnclaveID { + if curArg == "--"+CfgPolicyEnclaveID { kmEnclaveIDStr := os.Args[curArgIdx+1] kmEnclaveID := sgx.EnclaveIdentity{} if err := kmEnclaveID.UnmarshalHex(kmEnclaveIDStr); err != nil { @@ -145,17 +145,17 @@ func policyFromFlags() (*kmApi.PolicySGX, error) { enclaves[kmEnclaveID] = &kmApi.EnclavePolicySGX{ MayReplicate: []sgx.EnclaveIdentity{}, - MayQuery: make(map[signature.PublicKey][]sgx.EnclaveIdentity), + MayQuery: make(map[common.Namespace][]sgx.EnclaveIdentity), } for curArgIdx = curArgIdx + 2; curArgIdx < len(os.Args); curArgIdx++ { // Break, if the next enclave-id is caught. - if os.Args[curArgIdx] == "--"+cfgPolicyEnclaveID { + if os.Args[curArgIdx] == "--"+CfgPolicyEnclaveID { break } // Catch --keymanager.policy.may.replicate option - if os.Args[curArgIdx] == "--"+cfgPolicyMayReplicate { + if os.Args[curArgIdx] == "--"+CfgPolicyMayReplicate { replicateStr := os.Args[curArgIdx+1] for _, r := range strings.Split(replicateStr, ",") { replEnclaveID := sgx.EnclaveIdentity{} @@ -171,11 +171,11 @@ func policyFromFlags() (*kmApi.PolicySGX, error) { } // Catch --keymanager.policy.may.query option - if os.Args[curArgIdx] == "--"+cfgPolicyMayQuery { + if os.Args[curArgIdx] == "--"+CfgPolicyMayQuery { queryStr := os.Args[curArgIdx+1] qRuntimeIDStr := strings.Split(queryStr, "=")[0] - var qRuntimeID signature.PublicKey + var qRuntimeID common.Namespace if err := qRuntimeID.UnmarshalHex(qRuntimeIDStr); err != nil { logger.Error("failed to parse may-query runtime ID", "err", err, @@ -231,10 +231,10 @@ func doSignPolicy(cmd *cobra.Command, args []string) { os.Exit(1) } - if err = ioutil.WriteFile(viper.GetStringSlice(cfgPolicySigFile)[0], sigBytes, 0600); err != nil { + if err = ioutil.WriteFile(viper.GetStringSlice(CfgPolicySigFile)[0], sigBytes, 0600); err != nil { logger.Error("failed to write policy file signature", "err", err, - "cfgPolicySigFile", viper.GetStringSlice(cfgPolicySigFile), + "CfgPolicySigFile", viper.GetStringSlice(CfgPolicySigFile), ) os.Exit(1) } @@ -243,29 +243,29 @@ func doSignPolicy(cmd *cobra.Command, args []string) { func signPolicyFromFlags() (*signature.Signature, error) { var signer signature.Signer var err error - if viper.GetString(cfgPolicyKeyFile) != "" { + if viper.GetString(CfgPolicyKeyFile) != "" { var signerFactory signature.SignerFactory signerFactory, err = fileSigner.NewFactory("", signature.SignerUnknown) if err != nil { return nil, err } - signer, err = signerFactory.(*fileSigner.Factory).ForceLoad(viper.GetString(cfgPolicyKeyFile)) + signer, err = signerFactory.(*fileSigner.Factory).ForceLoad(viper.GetString(CfgPolicyKeyFile)) if err != nil { return nil, err } - } else if viper.GetUint(cfgPolicyTestKey) != 0 { + } else if viper.GetUint(CfgPolicyTestKey) != 0 { if !cmdFlags.DebugDontBlameOasis() { return nil, errors.New("refusing to use test keys for signing") } - if viper.GetUint(cfgPolicyTestKey) > uint(len(kmApi.TestSigners)) { + if viper.GetUint(CfgPolicyTestKey) > uint(len(kmApi.TestSigners)) { return nil, errors.New("test key index invalid") } - signer = kmApi.TestSigners[viper.GetUint(cfgPolicyTestKey)-1] + signer = kmApi.TestSigners[viper.GetUint(CfgPolicyTestKey)-1] } else { return nil, errors.New("no private key file or test key provided") } - policyBytes, err := ioutil.ReadFile(viper.GetString(cfgPolicyFile)) + policyBytes, err := ioutil.ReadFile(viper.GetString(CfgPolicyFile)) if err != nil { return nil, err } @@ -303,7 +303,7 @@ func doVerifyPolicy(cmd *cobra.Command, args []string) { } func verifyPolicyFromFlags() error { - policyBytes, err := ioutil.ReadFile(viper.GetString(cfgPolicyFile)) + policyBytes, err := ioutil.ReadFile(viper.GetString(CfgPolicyFile)) if err != nil { return err } @@ -322,8 +322,8 @@ func verifyPolicyFromFlags() error { // Check the signatures of the policy. Public key is taken from the PEM // signature file. - if !viper.GetBool(cfgPolicyIgnoreSig) { - for _, sigFile := range viper.GetStringSlice(cfgPolicySigFile) { + if !viper.GetBool(CfgPolicyIgnoreSig) { + for _, sigFile := range viper.GetStringSlice(CfgPolicySigFile) { policySigBytes, err := ioutil.ReadFile(sigFile) if err != nil { return err @@ -373,10 +373,10 @@ func doInitStatus(cmd *cobra.Command, args []string) { } c, _ := json.Marshal(s) - if err = ioutil.WriteFile(viper.GetString(cfgStatusFile), c, 0666); err != nil { + if err = ioutil.WriteFile(viper.GetString(CfgStatusFile), c, 0666); err != nil { logger.Error("failed to write key manager status json file", "err", err, - "cfgStatusFile", viper.GetString(cfgStatusFile), + "CfgStatusFile", viper.GetString(CfgStatusFile), ) os.Exit(1) } @@ -398,21 +398,21 @@ func doGenUpdate(cmd *cobra.Command, args []string) { // signatures. var signedPolicy kmApi.SignedPolicySGX - policyBytes, err := ioutil.ReadFile(viper.GetString(cfgPolicyFile)) + policyBytes, err := ioutil.ReadFile(viper.GetString(CfgPolicyFile)) if err != nil { logger.Error("failed to read policy file", "err", err, ) os.Exit(1) } - if err = json.Unmarshal(policyBytes, &signedPolicy.Policy); err != nil { + if err = cbor.Unmarshal(policyBytes, &signedPolicy.Policy); err != nil { logger.Error("failed to unmarshal policy file", "err", err, ) os.Exit(1) } - for _, sigFile := range viper.GetStringSlice(cfgPolicySigFile) { + for _, sigFile := range viper.GetStringSlice(CfgPolicySigFile) { var policySigBytes []byte if policySigBytes, err = ioutil.ReadFile(sigFile); err != nil { logger.Error("failed to read signature file", @@ -449,18 +449,18 @@ func doGenUpdate(cmd *cobra.Command, args []string) { func statusFromFlags() (*kmApi.Status, error) { var id common.Namespace - if err := id.UnmarshalHex(viper.GetString(cfgStatusID)); err != nil { + if err := id.UnmarshalHex(viper.GetString(CfgStatusID)); err != nil { logger.Error("failed to parse key manager status ID", "err", err, - "cfgStatusID", viper.GetString(cfgStatusID), + "CfgStatusID", viper.GetString(CfgStatusID), ) return nil, err } // Unmarshal KM policy and its signatures. var signedPolicy *kmApi.SignedPolicySGX - if viper.GetString(cfgPolicyFile) != "" { - pb, err := ioutil.ReadFile(viper.GetString(cfgPolicyFile)) + if viper.GetString(CfgPolicyFile) != "" { + pb, err := ioutil.ReadFile(viper.GetString(CfgPolicyFile)) if err != nil { return nil, err } @@ -473,7 +473,7 @@ func statusFromFlags() (*kmApi.Status, error) { Policy: *p, } - for _, sigFile := range viper.GetStringSlice(cfgPolicySigFile) { + for _, sigFile := range viper.GetStringSlice(CfgPolicySigFile) { sigBytes, err := ioutil.ReadFile(sigFile) if err != nil { return nil, err @@ -489,9 +489,9 @@ func statusFromFlags() (*kmApi.Status, error) { } checksum := []byte{} - if viper.GetString(cfgStatusChecksum) != "" { + if viper.GetString(CfgStatusChecksum) != "" { var err error - checksum, err = hex.DecodeString(viper.GetString(cfgStatusChecksum)) + checksum, err = hex.DecodeString(viper.GetString(CfgStatusChecksum)) if err != nil { return nil, err } @@ -500,18 +500,18 @@ func statusFromFlags() (*kmApi.Status, error) { } } - if viper.GetString(cfgStatusChecksum) != "" && !viper.GetBool(cfgStatusInitialized) { - return nil, fmt.Errorf("%s provided, but %s is false", cfgStatusChecksum, cfgStatusInitialized) + if viper.GetString(CfgStatusChecksum) != "" && !viper.GetBool(CfgStatusInitialized) { + return nil, fmt.Errorf("%s provided, but %s is false", CfgStatusChecksum, CfgStatusInitialized) } - if viper.GetString(cfgStatusChecksum) == "" && viper.GetBool(cfgStatusInitialized) { - return nil, fmt.Errorf("%s is true, but %s is not provided", cfgStatusInitialized, cfgStatusChecksum) + if viper.GetString(CfgStatusChecksum) == "" && viper.GetBool(CfgStatusInitialized) { + return nil, fmt.Errorf("%s is true, but %s is not provided", CfgStatusInitialized, CfgStatusChecksum) } return &kmApi.Status{ ID: id, - IsInitialized: viper.GetBool(cfgStatusInitialized), - IsSecure: viper.GetBool(cfgStatusSecure), + IsInitialized: viper.GetBool(CfgStatusInitialized), + IsSecure: viper.GetBool(CfgStatusSecure), Checksum: checksum, Policy: signedPolicy, }, nil @@ -519,28 +519,28 @@ func statusFromFlags() (*kmApi.Status, error) { func registerKMInitPolicyFlags(cmd *cobra.Command) { if !cmd.Flags().Parsed() { - cmd.Flags().Uint32(cfgPolicySerial, 0, "monotonically increasing number of the policy") - cmd.Flags().String(cfgPolicyID, "", "256-bit Runtime ID this policy is valid for in hex") - cmd.Flags().String(cfgPolicyEnclaveID, "", "512-bit Key Manager Enclave ID in hex (concatenated MRENCLAVE and MRSIGNER). Multiple Enclave IDs with corresponding permissions can be provided respectively.") - cmd.Flags().StringSlice(cfgPolicyMayReplicate, []string{}, "enclave_id1,enclave_id2... list of new enclaves which are allowed to access the master secret. Requires "+cfgPolicyEnclaveID) - cmd.Flags().StringToString(cfgPolicyMayQuery, map[string]string{}, "runtime_id=enclave_id1,enclave_id2... sets enclave query permission for runtime_id. Requires "+cfgPolicyEnclaveID) + cmd.Flags().Uint32(CfgPolicySerial, 0, "monotonically increasing number of the policy") + cmd.Flags().String(CfgPolicyID, "", "256-bit Runtime ID this policy is valid for in hex") + cmd.Flags().String(CfgPolicyEnclaveID, "", "512-bit Key Manager Enclave ID in hex (concatenated MRENCLAVE and MRSIGNER). Multiple Enclave IDs with corresponding permissions can be provided respectively.") + cmd.Flags().StringSlice(CfgPolicyMayReplicate, []string{}, "enclave_id1,enclave_id2... list of new enclaves which are allowed to access the master secret. Requires "+CfgPolicyEnclaveID) + cmd.Flags().StringToString(CfgPolicyMayQuery, map[string]string{}, "runtime_id=enclave_id1,enclave_id2... sets enclave query permission for runtime_id. Requires "+CfgPolicyEnclaveID) } cmd.Flags().AddFlagSet(policyFileFlag) for _, v := range []string{ - cfgPolicySerial, - cfgPolicyID, + CfgPolicySerial, + CfgPolicyID, } { _ = cmd.MarkFlagRequired(v) } for _, v := range []string{ - cfgPolicySerial, - cfgPolicyID, - cfgPolicyEnclaveID, - cfgPolicyMayReplicate, - cfgPolicyMayQuery, + CfgPolicySerial, + CfgPolicyID, + CfgPolicyEnclaveID, + CfgPolicyMayReplicate, + CfgPolicyMayQuery, } { _ = viper.BindPFlag(v, cmd.Flags().Lookup(v)) } @@ -548,9 +548,9 @@ func registerKMInitPolicyFlags(cmd *cobra.Command) { func registerKMSignPolicyFlags(cmd *cobra.Command) { if !cmd.Flags().Parsed() { - cmd.Flags().String(cfgPolicyKeyFile, "", "input file name containing client key") - cmd.Flags().Uint(cfgPolicyTestKey, 0, "index of test key to use (for debugging only) counting from 1") - _ = cmd.Flags().MarkHidden(cfgPolicyTestKey) + cmd.Flags().String(CfgPolicyKeyFile, "", "input file name containing client key") + cmd.Flags().Uint(CfgPolicyTestKey, 0, "index of test key to use (for debugging only) counting from 1") + _ = cmd.Flags().MarkHidden(CfgPolicyTestKey) } cmd.Flags().AddFlagSet(policyFileFlag) @@ -558,8 +558,8 @@ func registerKMSignPolicyFlags(cmd *cobra.Command) { cmd.Flags().AddFlagSet(cmdFlags.DebugDontBlameOasisFlag) for _, v := range []string{ - cfgPolicyKeyFile, - cfgPolicyTestKey, + CfgPolicyKeyFile, + CfgPolicyTestKey, } { _ = viper.BindPFlag(v, cmd.Flags().Lookup(v)) } @@ -567,7 +567,7 @@ func registerKMSignPolicyFlags(cmd *cobra.Command) { func registerKMVerifyPolicyFlags(cmd *cobra.Command) { if !cmd.Flags().Parsed() { - cmd.Flags().Bool(cfgPolicyIgnoreSig, false, "just check, if policy file is well formed and ignore signature file") + cmd.Flags().Bool(CfgPolicyIgnoreSig, false, "just check, if policy file is well formed and ignore signature file") } cmd.Flags().AddFlagSet(cmdFlags.VerboseFlags) @@ -575,7 +575,7 @@ func registerKMVerifyPolicyFlags(cmd *cobra.Command) { cmd.Flags().AddFlagSet(policySigFileFlag) for _, v := range []string{ - cfgPolicyIgnoreSig, + CfgPolicyIgnoreSig, } { _ = viper.BindPFlag(v, cmd.Flags().Lookup(v)) } @@ -583,28 +583,28 @@ func registerKMVerifyPolicyFlags(cmd *cobra.Command) { func registerKMInitStatusFlags(cmd *cobra.Command) { if !cmd.Flags().Parsed() { - cmd.Flags().String(cfgStatusID, "", "256-bit Runtime ID this status is valid for in hex") - cmd.Flags().String(cfgStatusFile, statusFilename, "JSON output file name of status to be written") - cmd.Flags().Bool(cfgStatusInitialized, false, "is key manager done initializing. Requires "+cfgStatusChecksum) - cmd.Flags().Bool(cfgStatusSecure, false, "is key manager secure") - cmd.Flags().String(cfgStatusChecksum, "", "key manager's master secret verification checksum in hex. Requires "+cfgStatusInitialized) + cmd.Flags().String(CfgStatusID, "", "256-bit Runtime ID this status is valid for in hex") + cmd.Flags().String(CfgStatusFile, statusFilename, "JSON output file name of status to be written") + cmd.Flags().Bool(CfgStatusInitialized, false, "is key manager done initializing. Requires "+CfgStatusChecksum) + cmd.Flags().Bool(CfgStatusSecure, false, "is key manager secure") + cmd.Flags().String(CfgStatusChecksum, "", "key manager's master secret verification checksum in hex. Requires "+CfgStatusInitialized) } cmd.Flags().AddFlagSet(policyFileFlag) cmd.Flags().AddFlagSet(policySigFileFlag) for _, v := range []string{ - cfgStatusID, + CfgStatusID, } { _ = cmd.MarkFlagRequired(v) } for _, v := range []string{ - cfgStatusID, - cfgStatusFile, - cfgStatusInitialized, - cfgStatusSecure, - cfgStatusChecksum, + CfgStatusID, + CfgStatusFile, + CfgStatusInitialized, + CfgStatusSecure, + CfgStatusChecksum, } { _ = viper.BindPFlag(v, cmd.Flags().Lookup(v)) } @@ -612,8 +612,8 @@ func registerKMInitStatusFlags(cmd *cobra.Command) { // Register registers the keymanager sub-command and all of it's children. func Register(parentCmd *cobra.Command) { - policyFileFlag.String(cfgPolicyFile, policyFilename, "file name of policy in CBOR format") - policySigFileFlag.StringSlice(cfgPolicySigFile, []string{policyFilename + ".sign"}, "file name(s) containing policy signature") + policyFileFlag.String(CfgPolicyFile, policyFilename, "file name of policy in CBOR format") + policySigFileFlag.StringSlice(CfgPolicySigFile, []string{policyFilename + ".sign"}, "file name(s) containing policy signature") _ = viper.BindPFlags(policyFileFlag) _ = viper.BindPFlags(policySigFileFlag) @@ -633,5 +633,9 @@ func Register(parentCmd *cobra.Command) { registerKMVerifyPolicyFlags(verifyPolicyCmd) registerKMInitStatusFlags(initStatusCmd) + genUpdateCmd.Flags().AddFlagSet(policyFileFlag) + genUpdateCmd.Flags().AddFlagSet(policySigFileFlag) + genUpdateCmd.Flags().AddFlagSet(cmdConsensus.TxFlags) + parentCmd.AddCommand(keyManagerCmd) } diff --git a/go/oasis-test-runner/oasis/cli/cli.go b/go/oasis-test-runner/oasis/cli/cli.go index 7c8859f5f15..49c4dfabf80 100644 --- a/go/oasis-test-runner/oasis/cli/cli.go +++ b/go/oasis-test-runner/oasis/cli/cli.go @@ -29,8 +29,9 @@ func (b *helpersBase) runSubCommandWithOutput(name string, args []string) (bytes // Helpers are the oasis-node cli helpers. type Helpers struct { - Consensus *ConsensusHelpers - Registry *RegistryHelpers + Consensus *ConsensusHelpers + Registry *RegistryHelpers + Keymanager *KeymanagerHelpers } // New creates new oasis-node cli helpers. @@ -42,8 +43,9 @@ func New(env *env.Env, net *oasis.Network, logger *logging.Logger) *Helpers { } return &Helpers{ - Consensus: &ConsensusHelpers{base}, - Registry: &RegistryHelpers{base}, + Consensus: &ConsensusHelpers{base}, + Registry: &RegistryHelpers{base}, + Keymanager: &KeymanagerHelpers{base}, } } diff --git a/go/oasis-test-runner/oasis/cli/keymanager.go b/go/oasis-test-runner/oasis/cli/keymanager.go new file mode 100644 index 00000000000..427543d4c5f --- /dev/null +++ b/go/oasis-test-runner/oasis/cli/keymanager.go @@ -0,0 +1,113 @@ +package cli + +import ( + "fmt" + "strconv" + "strings" + + "github.com/oasislabs/oasis-core/go/common" + "github.com/oasislabs/oasis-core/go/common/sgx" + keymanager "github.com/oasislabs/oasis-core/go/keymanager/api" + cmdCommon "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common" + cmdConsensus "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/consensus" + "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/flags" + cmdKM "github.com/oasislabs/oasis-core/go/oasis-node/cmd/keymanager" +) + +// KeymanagerHelpers contains the oasis-node keymanager CLI helpers. +type KeymanagerHelpers struct { + *helpersBase +} + +// InitPolicy generates the KM policy file. +func (k *KeymanagerHelpers) InitPolicy(runtimeID common.Namespace, serial uint32, policies map[sgx.EnclaveIdentity]*keymanager.EnclavePolicySGX, polPath string) error { + k.logger.Info("initing KM policy", + "policy_path", polPath, + "serial", serial, + "num_policies", len(policies), + ) + + args := []string{ + "keymanager", "init_policy", + "--" + cmdKM.CfgPolicyFile, polPath, + "--" + cmdKM.CfgPolicyID, runtimeID.String(), + "--" + cmdKM.CfgPolicySerial, strconv.FormatUint(uint64(serial), 10), + } + for enclave, policy := range policies { + args = append(args, "--"+cmdKM.CfgPolicyEnclaveID) + args = append(args, enclave.String()) + if len(policy.MayQuery) > 0 { + for rtID, encIDs := range policy.MayQuery { + args = append(args, "--"+cmdKM.CfgPolicyMayQuery) + var encIDstrs []string + for _, eid := range encIDs { + encIDstrs = append(encIDstrs, eid.String()) + } + args = append(args, rtID.String()+"="+strings.Join(encIDstrs, ",")) + } + } + if len(policy.MayReplicate) > 0 { + args = append(args, "--"+cmdKM.CfgPolicyMayReplicate) + var encIDstrs []string + for _, eid := range policy.MayReplicate { + encIDstrs = append(encIDstrs, eid.String()) + } + args = append(args, strings.Join(encIDstrs, ",")) + } + } + if err := k.runSubCommand("keymanager-init_policy", args); err != nil { + return fmt.Errorf("failed to init KM policy: %w", err) + } + return nil +} + +// SignPolicy signs the KM policy file using the given test key ("1", "2", or "3"). +func (k *KeymanagerHelpers) SignPolicy(testKey string, polPath string, polSigPath string) error { + k.logger.Info("signing KM policy", + "policy_path", polPath, + "policy_signature_path", polSigPath, + "test_key", testKey, + ) + + args := []string{ + "keymanager", "sign_policy", + "--" + flags.CfgDebugDontBlameOasis, + "--" + cmdCommon.CfgDebugAllowTestKeys, + "--" + cmdKM.CfgPolicyFile, polPath, + "--" + cmdKM.CfgPolicySigFile, polSigPath, + "--" + cmdKM.CfgPolicyTestKey, testKey, + } + if err := k.runSubCommand("keymanager-sign_policy", args); err != nil { + return fmt.Errorf("failed to sign KM policy: %w", err) + } + return nil +} + +// GenUpdate generates the update KM policy transaction. +func (k *KeymanagerHelpers) GenUpdate(nonce uint64, polPath string, polSigPaths []string, txPath string) error { + k.logger.Info("generating KM policy update", + "policy_path", polPath, + "policy_signature_paths", polSigPaths, + "transaction_path", txPath, + ) + + args := []string{ + "keymanager", "gen_update", + "--" + cmdConsensus.CfgTxNonce, strconv.FormatUint(nonce, 10), + "--" + cmdConsensus.CfgTxFile, txPath, + "--" + cmdConsensus.CfgTxFeeAmount, strconv.Itoa(0), // TODO: Make fee configurable. + "--" + cmdConsensus.CfgTxFeeGas, strconv.Itoa(10000), // TODO: Make fee configurable. + "--" + cmdKM.CfgPolicyFile, polPath, + "--" + flags.CfgDebugDontBlameOasis, + "--" + cmdCommon.CfgDebugAllowTestKeys, + "--" + flags.CfgDebugTestEntity, + "--" + flags.CfgGenesisFile, k.net.GenesisPath(), + } + for _, sigPath := range polSigPaths { + args = append(args, "--"+cmdKM.CfgPolicySigFile, sigPath) + } + if err := k.runSubCommand("keymanager-gen_update", args); err != nil { + return fmt.Errorf("failed to generate KM update transaction: %w", err) + } + return nil +} diff --git a/go/oasis-test-runner/oasis/keymanager.go b/go/oasis-test-runner/oasis/keymanager.go index bd85464c417..36d8f1b4c4d 100644 --- a/go/oasis-test-runner/oasis/keymanager.go +++ b/go/oasis-test-runner/oasis/keymanager.go @@ -78,6 +78,10 @@ func (km *Keymanager) Start() error { } func (km *Keymanager) provisionGenesis() error { + if km.runtime.excludeFromGenesis { + return nil + } + // Provision status and policy. We can only provision this here as we need // a list of runtimes allowed to query the key manager. statusArgs := []string{ @@ -174,6 +178,10 @@ func (km *Keymanager) provisionGenesis() error { } func (km *Keymanager) toGenesisArgs() []string { + if km.runtime.excludeFromGenesis { + return nil + } + return []string{ "--keymanager", filepath.Join(km.dir.String(), kmStatusFile), } diff --git a/go/oasis-test-runner/oasis/runtime.go b/go/oasis-test-runner/oasis/runtime.go index 6acea7711d4..76a6460a4e4 100644 --- a/go/oasis-test-runner/oasis/runtime.go +++ b/go/oasis-test-runner/oasis/runtime.go @@ -76,6 +76,22 @@ func (rt *Runtime) ID() common.Namespace { return rt.id } +// Kind returns the runtime kind. +func (rt *Runtime) Kind() registry.RuntimeKind { + return rt.kind +} + +// GetEnclaveIdentity returns the runtime's enclave ID. +func (rt *Runtime) GetEnclaveIdentity() *sgx.EnclaveIdentity { + if rt.mrEnclave != nil && rt.mrSigner != nil { + return &sgx.EnclaveIdentity{ + MrEnclave: *rt.mrEnclave, + MrSigner: *rt.mrSigner, + } + } + return nil +} + func (rt *Runtime) toGenesisArgs() []string { if rt.excludeFromGenesis { return []string{} diff --git a/go/oasis-test-runner/scenario/e2e/runtime_dynamic.go b/go/oasis-test-runner/scenario/e2e/runtime_dynamic.go index 54fd011de61..5a3c997ddf9 100644 --- a/go/oasis-test-runner/scenario/e2e/runtime_dynamic.go +++ b/go/oasis-test-runner/scenario/e2e/runtime_dynamic.go @@ -6,11 +6,14 @@ import ( "path/filepath" "time" + "github.com/oasislabs/oasis-core/go/common" "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/quantity" + "github.com/oasislabs/oasis-core/go/common/sgx" consensus "github.com/oasislabs/oasis-core/go/consensus/api" "github.com/oasislabs/oasis-core/go/consensus/api/transaction" epochtime "github.com/oasislabs/oasis-core/go/epochtime/api" + keymanager "github.com/oasislabs/oasis-core/go/keymanager/api" "github.com/oasislabs/oasis-core/go/oasis-test-runner/env" "github.com/oasislabs/oasis-core/go/oasis-test-runner/oasis" "github.com/oasislabs/oasis-core/go/oasis-test-runner/oasis/cli" @@ -49,12 +52,7 @@ func (sc *runtimeDynamicImpl) Fixture() (*oasis.NetworkFixture, error) { // Avoid unexpected blocks. f.Network.EpochtimeMock = true // Exclude all runtimes from genesis as we will register those dynamically. - for i, rt := range f.Runtimes { - // TODO: This should not be needed once dynamic keymanager policy document registration - // is supported (see oasis-core#2516). - if rt.Kind != registry.KindCompute { - continue - } + for i := range f.Runtimes { f.Runtimes[i].ExcludeFromGenesis = true } @@ -87,9 +85,8 @@ func (sc *runtimeDynamicImpl) Run(childEnv *env.Env) error { // nolint: gocyclo return err } - // TODO: Once dynamic key manager registration is supported, waiting for keymanagers - // shouldn't be needed. - numNodes := len(sc.net.Validators()) + len(sc.net.Keymanagers()) + // Wait for enough nodes to register. + numNodes := len(sc.net.Validators()) sc.logger.Info("waiting for (some) nodes to register", "num_nodes", numNodes, ) @@ -103,14 +100,99 @@ func (sc *runtimeDynamicImpl) Run(childEnv *env.Env) error { // nolint: gocyclo return err } - // TODO: Register a new key manager runtime and status (see oasis-core#2516). + // Nonce used for transactions (increase this by 1 after each transaction). + var nonce uint64 + + // Register a new keymanager runtime. + kmRt := sc.net.Runtimes()[0] + kmRtDesc := kmRt.ToRuntimeDescriptor() + kmTxPath := filepath.Join(childEnv.Dir(), "register_km_runtime.json") + if err := cli.Registry.GenerateRegisterRuntimeTx(nonce, kmRtDesc, kmTxPath, ""); err != nil { + return fmt.Errorf("failed to generate register KM runtime tx: %w", err) + } + nonce++ + if err := cli.Consensus.SubmitTx(kmTxPath); err != nil { + return fmt.Errorf("failed to register KM runtime: %w", err) + } + + // Generate and update the new keymanager runtime's policy. + kmPolicyPath := filepath.Join(childEnv.Dir(), "km_policy.cbor") + kmPolicySig1Path := filepath.Join(childEnv.Dir(), "km_policy_sig1.pem") + kmPolicySig2Path := filepath.Join(childEnv.Dir(), "km_policy_sig2.pem") + kmPolicySig3Path := filepath.Join(childEnv.Dir(), "km_policy_sig3.pem") + kmUpdateTxPath := filepath.Join(childEnv.Dir(), "km_gen_update.json") + sc.logger.Info("building KM SGX policy enclave policies map") + enclavePolicies := make(map[sgx.EnclaveIdentity]*keymanager.EnclavePolicySGX) + kmRtEncID := kmRt.GetEnclaveIdentity() + var havePolicy bool + if kmRtEncID != nil { + enclavePolicies[*kmRtEncID] = &keymanager.EnclavePolicySGX{} + enclavePolicies[*kmRtEncID].MayQuery = make(map[common.Namespace][]sgx.EnclaveIdentity) + enclavePolicies[*kmRtEncID].MayReplicate = []sgx.EnclaveIdentity{} + for _, rt := range sc.net.Runtimes() { + if rt.Kind() != registry.KindCompute { + continue + } + if eid := rt.GetEnclaveIdentity(); eid != nil { + enclavePolicies[*kmRtEncID].MayQuery[rt.ID()] = []sgx.EnclaveIdentity{*eid} + // This is set only in SGX mode. + havePolicy = true + } + } + } + sc.logger.Info("initing KM policy") + if err := cli.Keymanager.InitPolicy(kmRt.ID(), 1, enclavePolicies, kmPolicyPath); err != nil { + return err + } + sc.logger.Info("signing KM policy") + if err := cli.Keymanager.SignPolicy("1", kmPolicyPath, kmPolicySig1Path); err != nil { + return err + } + if err := cli.Keymanager.SignPolicy("2", kmPolicyPath, kmPolicySig2Path); err != nil { + return err + } + if err := cli.Keymanager.SignPolicy("3", kmPolicyPath, kmPolicySig3Path); err != nil { + return err + } + if havePolicy { + // In SGX mode, we can update the policy as intended. + sc.logger.Info("updating KM policy") + if err := cli.Keymanager.GenUpdate(nonce, kmPolicyPath, []string{kmPolicySig1Path, kmPolicySig2Path, kmPolicySig3Path}, kmUpdateTxPath); err != nil { + return err + } + nonce++ + if err := cli.Consensus.SubmitTx(kmUpdateTxPath); err != nil { + return fmt.Errorf("failed to update KM policy: %w", err) + } + } else { + // In non-SGX mode, the policy update fails with a policy checksum + // mismatch (the non-SGX KM returns an empty policy), so we need to + // do an epoch transition instead (to complete the KM runtime + // registration). + if err := sc.epochTransition(ctx); err != nil { + return err + } + } + + // Wait for enough nodes to register, then make another epoch transition. + numNodes += len(sc.net.Keymanagers()) + sc.logger.Info("waiting for (some) nodes to register (again)", + "num_nodes", numNodes, + ) + if err := sc.net.Controller().WaitNodesRegistered(ctx, numNodes); err != nil { + return err + } + if err := sc.epochTransition(ctx); err != nil { + return err + } // Register a new compute runtime. compRt := sc.net.Runtimes()[1].ToRuntimeDescriptor() txPath := filepath.Join(childEnv.Dir(), "register_compute_runtime.json") - if err := cli.Registry.GenerateRegisterRuntimeTx(0, compRt, txPath, ""); err != nil { - return fmt.Errorf("failed to generate register runtime tx: %w", err) + if err := cli.Registry.GenerateRegisterRuntimeTx(nonce, compRt, txPath, ""); err != nil { + return fmt.Errorf("failed to generate register compute runtime tx: %w", err) } + nonce++ if err := cli.Consensus.SubmitTx(txPath); err != nil { return fmt.Errorf("failed to register compute runtime: %w", err) } @@ -218,10 +300,11 @@ func (sc *runtimeDynamicImpl) Run(childEnv *env.Env) error { // nolint: gocyclo entSigner := sc.net.Entities()[0].Signer() var oneShare quantity.Quantity _ = oneShare.FromUint64(1) - tx := staking.NewReclaimEscrowTx(1, &transaction.Fee{Gas: 10000}, &staking.ReclaimEscrow{ + tx := staking.NewReclaimEscrowTx(nonce, &transaction.Fee{Gas: 10000}, &staking.ReclaimEscrow{ Account: entSigner.Public(), Shares: oneShare, }) + nonce++ sigTx, err := transaction.Sign(entSigner, tx) if err != nil { return fmt.Errorf("failed to sign reclaim: %w", err) @@ -312,10 +395,11 @@ func (sc *runtimeDynamicImpl) Run(childEnv *env.Env) error { // nolint: gocyclo sc.logger.Info("escrowing stake back") var enoughTokens quantity.Quantity _ = enoughTokens.FromUint64(100_000) - tx = staking.NewAddEscrowTx(2, &transaction.Fee{Gas: 10000}, &staking.Escrow{ + tx = staking.NewAddEscrowTx(nonce, &transaction.Fee{Gas: 10000}, &staking.Escrow{ Account: entSigner.Public(), Tokens: enoughTokens, }) + nonce++ // nolint: ineffassign sigTx, err = transaction.Sign(entSigner, tx) if err != nil { return fmt.Errorf("failed to sign escrow: %w", err)