keymanager runtime upgrade e2e test

oasisprotocol · May 20, 2020 · b8d456f · b8d456f
1 parent 08e237f
commit b8d456f
Show file tree

Hide file tree

Showing 27 changed files with 591 additions and 108 deletions.
diff --git a/.buildkite/benchmarks.pipeline.yml b/.buildkite/benchmarks.pipeline.yml
@@ -97,14 +97,17 @@ steps:
     command:
       - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keymanager
       - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keyvalue
+      - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keymanager-upgrade
 
       # Upload the built artifacts.
       - cd /var/tmp/artifacts/sgx/x86_64-fortanix-unknown-sgx/debug
       - buildkite-agent artifact upload simple-keymanager.sgxs
       - buildkite-agent artifact upload simple-keyvalue.sgxs
+      - buildkite-agent artifact upload simple-keymanager-upgrade.sgxs
       - cd /var/tmp/artifacts/default/debug
       - buildkite-agent artifact upload simple-keymanager
       - buildkite-agent artifact upload simple-keyvalue
+      - buildkite-agent artifact upload simple-keymanager-upgrade
     agents:
       buildkite_agent_size: large
     plugins:

diff --git a/.buildkite/code.pipeline.yml b/.buildkite/code.pipeline.yml
@@ -159,14 +159,17 @@ steps:
     command:
       - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keymanager
       - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keyvalue
+      - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keymanager-upgrade
 
       # Upload the built artifacts.
       - cd /var/tmp/artifacts/sgx/x86_64-fortanix-unknown-sgx/debug
       - buildkite-agent artifact upload simple-keymanager.sgxs
       - buildkite-agent artifact upload simple-keyvalue.sgxs
+      - buildkite-agent artifact upload simple-keymanager-upgrade.sgxs
       - cd /var/tmp/artifacts/default/debug
       - buildkite-agent artifact upload simple-keymanager
       - buildkite-agent artifact upload simple-keyvalue
+      - buildkite-agent artifact upload simple-keymanager-upgrade
     agents:
       buildkite_agent_size: large
     retry:

diff --git a/.buildkite/longtests.pipeline.yml b/.buildkite/longtests.pipeline.yml
@@ -81,14 +81,17 @@ steps:
     command:
       - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keymanager
       - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keyvalue
+      - .buildkite/rust/build_runtime.sh tests/runtimes/simple-keymanager-upgrade
 
       # Upload the built artifacts.
       - cd /var/tmp/artifacts/sgx/x86_64-fortanix-unknown-sgx/debug
       - buildkite-agent artifact upload simple-keymanager.sgxs
       - buildkite-agent artifact upload simple-keyvalue.sgxs
+      - buildkite-agent artifact upload simple-keymanager-upgrade.sgxs
       - cd /var/tmp/artifacts/default/debug
       - buildkite-agent artifact upload simple-keymanager
       - buildkite-agent artifact upload simple-keyvalue
+      - buildkite-agent artifact upload simple-keymanager-upgrade
     agents:
       buildkite_agent_size: large
     plugins:

diff --git a/.buildkite/scripts/download_e2e_test_artifacts.sh b/.buildkite/scripts/download_e2e_test_artifacts.sh
@@ -24,6 +24,10 @@ download_artifact oasis-core-runtime-loader target/default/debug 755
 download_artifact simple-keymanager.sgxs target/sgx/x86_64-fortanix-unknown-sgx/debug 755
 download_artifact simple-keymanager target/default/debug 755
 
+# Simple Key manager runtime used in keymenager upgrade test.
+download_artifact simple-keymanager-upgrade.sgxs target/sgx/x86_64-fortanix-unknown-sgx/debug 755
+download_artifact simple-keymanager-upgrade target/default/debug 755
+
 # Test simple-keyvalue runtime and clients.
 download_artifact test-long-term-client target/default/debug 755
 download_artifact simple-keyvalue-client target/default/debug 755

diff --git a/.changelog/2517.bugfix.md b/.changelog/2517.bugfix.md
@@ -0,0 +1,6 @@
+go/worker/keymanager: retry initialization in case of failure
+
+The keymanager worker registers only after the initialization either fails or
+succeeds. In case the worker needs to replicate the first initialization will
+always fail, since other nodes' access control prevent it from replicating. In
+that case the initialization should be retried.
diff --git a/.changelog/2517.feature.md b/.changelog/2517.feature.md
@@ -0,0 +1 @@
+e2e/tests: added keymanager runtime upgrade test
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -11,6 +11,8 @@ members = [
     # Test runtimes.
     "tests/runtimes/simple-keyvalue",
     "tests/runtimes/simple-keymanager",
+    "tests/runtimes/simple-keymanager-upgrade",
+
     # Test clients.
     "tests/clients/simple-keyvalue",
     "tests/clients/simple-keyvalue-enc",

diff --git a/Makefile b/Makefile
@@ -4,7 +4,8 @@ include common.mk
 
 # List of runtimes to build.
 RUNTIMES := tests/runtimes/simple-keyvalue \
-	tests/runtimes/simple-keymanager
+	tests/runtimes/simple-keymanager \
+	tests/runtimes/simple-keymanager-upgrade
 
 # Set all target as the default target.
 all: build

diff --git a/go/oasis-net-runner/fixtures/default.go b/go/oasis-net-runner/fixtures/default.go
@@ -67,7 +67,7 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) {
 				Kind:       registry.KindKeyManager,
 				Entity:     0,
 				Keymanager: -1,
-				Binary:     viper.GetString(cfgKeymanagerBinary),
+				Binaries:   viper.GetStringSlice(cfgKeymanagerBinary),
 				AdmissionPolicy: registry.RuntimeAdmissionPolicy{
 					AnyNode: &registry.AnyNodeRuntimeAdmissionPolicy{},
 				},
@@ -78,7 +78,7 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) {
 				Kind:       registry.KindCompute,
 				Entity:     0,
 				Keymanager: 0,
-				Binary:     viper.GetString(cfgRuntimeBinary),
+				Binaries:   viper.GetStringSlice(cfgRuntimeBinary),
 				Executor: registry.ExecutorParameters{
 					GroupSize:       2,
 					GroupBackupSize: 1,

diff --git a/go/oasis-test-runner/oasis/args.go b/go/oasis-test-runner/oasis/args.go
@@ -454,9 +454,9 @@ func (args *argBuilder) appendRuntimePruner(p *RuntimePrunerCfg) *argBuilder {
 	return args
 }
 
-func (args *argBuilder) appendComputeNodeRuntime(rt *Runtime) *argBuilder {
+func (args *argBuilder) appendComputeNodeRuntime(rt *Runtime, binaryIdx int) *argBuilder {
 	args = args.runtimeSupported(rt.id).
-		workerRuntimePath(rt.id, rt.binary).
+		workerRuntimePath(rt.id, rt.binaries[binaryIdx]).
 		appendRuntimePruner(&rt.pruner)
 	return args
 }
@@ -494,7 +494,7 @@ func (args *argBuilder) byzantineFakeSGX() *argBuilder {
 
 func (args *argBuilder) byzantineVersionFakeEnclaveID(rt *Runtime) *argBuilder {
 	eid := sgx.EnclaveIdentity{
-		MrEnclave: *rt.mrEnclave,
+		MrEnclave: *rt.mrEnclaves[0],
 		MrSigner:  *rt.mrSigner,
 	}
 	args.vec = append(args.vec, "--"+byzantine.CfgVersionFakeEnclaveID, eid.String())

diff --git a/go/oasis-test-runner/oasis/compute.go b/go/oasis-test-runner/oasis/compute.go
@@ -95,7 +95,8 @@ func (worker *Compute) startNode() error {
 		if v.kind != registry.KindCompute {
 			continue
 		}
-		args = args.appendComputeNodeRuntime(v)
+		// XXX: could support configurable binary idx if ever needed.
+		args = args.appendComputeNodeRuntime(v, 0)
 	}
 
 	if err := worker.net.startOasisNode(&worker.Node, nil, args); err != nil {

diff --git a/go/oasis-test-runner/oasis/fixture.go b/go/oasis-test-runner/oasis/fixture.go
@@ -188,9 +188,9 @@ type RuntimeFixture struct { // nolint: maligned
 	Entity     int                  `json:"entity"`
 	Keymanager int                  `json:"keymanager"`
 
-	Binary       string `json:"binary"`
-	GenesisState string `json:"genesis_state"`
-	GenesisRound uint64 `json:"genesis_round"`
+	Binaries     []string `json:"binaries"`
+	GenesisState string   `json:"genesis_state"`
+	GenesisRound uint64   `json:"genesis_round"`
 
 	Executor     registry.ExecutorParameters     `json:"executor"`
 	Merge        registry.MergeParameters        `json:"merge"`
@@ -235,7 +235,7 @@ func (f *RuntimeFixture) Create(netFixture *NetworkFixture, net *Network) (*Runt
 		TxnScheduler:       f.TxnScheduler,
 		Storage:            f.Storage,
 		AdmissionPolicy:    f.AdmissionPolicy,
-		Binary:             f.Binary,
+		Binaries:           f.Binaries,
 		GenesisState:       f.GenesisState,
 		GenesisRound:       f.GenesisRound,
 		Pruner:             f.Pruner,
@@ -271,6 +271,8 @@ type KeymanagerFixture struct {
 	AllowEarlyTermination bool `json:"allow_early_termination"`
 	AllowErrorTermination bool `json:"allow_error_termination"`
 
+	NoAutoStart bool `json:"no_auto_start,omitempty"`
+
 	Sentries []int `json:"sentries,omitempty"`
 
 	// Consensus contains configuration for the consensus backend.
@@ -300,6 +302,7 @@ func (f *KeymanagerFixture) Create(net *Network) (*Keymanager, error) {
 			AllowErrorTermination:      f.AllowErrorTermination,
 			LogWatcherHandlerFactories: f.LogWatcherHandlerFactories,
 			Consensus:                  f.Consensus,
+			NoAutoStart:                f.NoAutoStart,
 		},
 		Runtime:       runtime,
 		Entity:        entity,

diff --git a/go/oasis-test-runner/oasis/keymanager.go b/go/oasis-test-runner/oasis/keymanager.go
@@ -56,16 +56,22 @@ func (pol *KeymanagerPolicy) provision() error {
 			"--" + kmCmd.CfgPolicyFile, policyPath,
 			"--" + kmCmd.CfgPolicyID, pol.runtime.id.String(),
 			"--" + kmCmd.CfgPolicySerial, strconv.Itoa(pol.serial),
-			"--" + kmCmd.CfgPolicyEnclaveID, pol.runtime.mrEnclave.String() + pol.runtime.mrSigner.String(),
+		}
+		for _, mrEnclave := range pol.runtime.mrEnclaves {
+			policyArgs = append(policyArgs, []string{
+				"--" + kmCmd.CfgPolicyEnclaveID, mrEnclave.String() + pol.runtime.mrSigner.String(),
+			}...)
 		}
 
 		for _, rt := range pol.net.runtimes {
 			if rt.teeHardware == node.TEEHardwareInvalid || rt.kind != registry.KindCompute {
 				continue
 			}
 
-			arg := fmt.Sprintf("%s=%s%s", rt.id, rt.mrEnclave, rt.mrSigner)
-			policyArgs = append(policyArgs, "--"+kmCmd.CfgPolicyMayQuery, arg)
+			for _, mrEnclave := range rt.mrEnclaves {
+				arg := fmt.Sprintf("%s=%s%s", rt.id, mrEnclave, rt.mrSigner)
+				policyArgs = append(policyArgs, "--"+kmCmd.CfgPolicyMayQuery, arg)
+			}
 		}
 
 		w, err := pol.dir.NewLogWriter("provision-policy.log")
@@ -263,7 +269,8 @@ func (km *Keymanager) startNode() error {
 		workerClientPort(km.workerClientPort).
 		workerRuntimeProvisioner(workerCommon.RuntimeProvisionerSandboxed).
 		workerRuntimeSGXLoader(km.net.cfg.RuntimeSGXLoaderBinary).
-		workerRuntimePath(km.runtime.id, km.runtime.binary).
+		// XXX: could support configurable binary idx if ever needed.
+		workerRuntimePath(km.runtime.id, km.runtime.binaries[0]).
 		workerKeymanagerEnabled().
 		workerKeymanagerRuntimeID(km.runtime.id).
 		appendNetwork(km.net).
@@ -325,6 +332,7 @@ func (net *Network) NewKeymanager(cfg *KeymanagerCfg) (*Keymanager, error) {
 			disableDefaultLogWatcherHandlerFactories: cfg.DisableDefaultLogWatcherHandlerFactories,
 			logWatcherHandlerFactories:               cfg.LogWatcherHandlerFactories,
 			consensus:                                cfg.Consensus,
+			noAutoStart:                              cfg.NoAutoStart,
 		},
 		runtime:          cfg.Runtime,
 		entity:           cfg.Entity,
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		e2e/tests: added keymanager runtime upgrade test