go/runtime/client: Perform local CheckTx when a hosted runtime exists

.buildkite/scripts/test_e2e.sh

@@ -17,12 +17,6 @@ WORKDIR=$PWD
 #################
 # Run test suite.
 #################
-# Determine correct runtime to use for SGX.
-runtime_target="default"
-if [[ "${OASIS_TEE_HARDWARE:-""}" == "intel-sgx" ]]; then
-    runtime_target="sgx/x86_64-fortanix-unknown-sgx"
-fi
-
 # We need a directory in the workdir so that Buildkite can fetch artifacts.
 if [[ "${BUILDKITE:-""}" != "" ]]; then
     mkdir -p ${TEST_BASE_DIR:-$PWD}/e2e
@@ -55,7 +49,8 @@ ${test_runner_binary} \
     --basedir.no_cleanup \
     --e2e.node.binary ${node_binary} \
     --e2e/runtime.client.binary_dir ${WORKDIR}/target/default/debug \
-    --e2e/runtime.runtime.binary_dir ${WORKDIR}/target/${runtime_target}/debug \
+    --e2e/runtime.runtime.binary_dir.default ${WORKDIR}/target/default/debug \
+    --e2e/runtime.runtime.binary_dir.intel-sgx ${WORKDIR}/target/sgx/x86_64-fortanix-unknown-sgx/debug \
     --e2e/runtime.runtime.loader ${WORKDIR}/target/default/debug/oasis-core-runtime-loader \
     --e2e/runtime.tee_hardware ${OASIS_TEE_HARDWARE:-""} \
     --e2e/runtime.ias.mock=${ias_mock} \

.changelog/3653.feature.md

@@ -0,0 +1 @@
+go/runtime/client: Perform local CheckTx when a hosted runtime exists

go/oasis-net-runner/fixtures/default.go

@@ -94,7 +94,9 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) {
 				Kind:       registry.KindKeyManager,
 				Entity:     0,
 				Keymanager: -1,
-				Binaries:   viper.GetStringSlice(cfgKeymanagerBinary),
+				Binaries: map[node.TEEHardware][]string{
+					tee: viper.GetStringSlice(cfgKeymanagerBinary),
+				},
 				AdmissionPolicy: registry.RuntimeAdmissionPolicy{
 					AnyNode: &registry.AnyNodeRuntimeAdmissionPolicy{},
 				},
@@ -105,7 +107,9 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) {
 				Kind:       registry.KindCompute,
 				Entity:     0,
 				Keymanager: 0,
-				Binaries:   viper.GetStringSlice(cfgRuntimeBinary),
+				Binaries: map[node.TEEHardware][]string{
+					tee: viper.GetStringSlice(cfgRuntimeBinary),
+				},
 				Executor: registry.ExecutorParameters{
 					GroupSize:       2,
 					GroupBackupSize: 1,

go/oasis-test-runner/oasis/args.go

@@ -14,6 +14,7 @@ import (
 	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/crash"
 	commonGrpc "github.com/oasisprotocol/oasis-core/go/common/grpc"
+	"github.com/oasisprotocol/oasis-core/go/common/node"
 	"github.com/oasisprotocol/oasis-core/go/common/sgx"
 	"github.com/oasisprotocol/oasis-core/go/consensus/tendermint"
 	"github.com/oasisprotocol/oasis-core/go/consensus/tendermint/abci"
@@ -534,9 +535,9 @@ func (args *argBuilder) appendRuntimePruner(p *RuntimePrunerCfg) *argBuilder {
 	return args
 }
 
-func (args *argBuilder) appendComputeNodeRuntime(rt *Runtime, binaryIdx int) *argBuilder {
+func (args *argBuilder) appendHostedRuntime(rt *Runtime, tee node.TEEHardware, binaryIdx int) *argBuilder {
 	args = args.runtimeSupported(rt.id).
-		runtimePath(rt.id, rt.binaries[binaryIdx]).
+		runtimePath(rt.id, rt.binaries[tee][binaryIdx]).
 		appendRuntimePruner(&rt.pruner)
 	return args
 }

go/oasis-test-runner/oasis/client.go

@@ -3,6 +3,7 @@ package oasis
 import (
 	"fmt"
 
+	"github.com/oasisprotocol/oasis-core/go/common/node"
 	registry "github.com/oasisprotocol/oasis-core/go/registry/api"
 )
 
@@ -44,8 +45,8 @@ func (client *Client) startNode() error {
 		if v.kind != registry.KindCompute {
 			continue
 		}
-		args = args.runtimeSupported(v.id).
-			appendRuntimePruner(&v.pruner)
+		// XXX: could support configurable binary idx if ever needed.
+		args = args.appendHostedRuntime(v, node.TEEHardwareInvalid, 0)
 	}
 
 	if err := client.net.startOasisNode(&client.Node, nil, args); err != nil {

go/oasis-test-runner/oasis/compute.go

@@ -112,7 +112,7 @@ func (worker *Compute) startNode() error {
 	for _, idx := range worker.runtimes {
 		v := worker.net.runtimes[idx]
 		// XXX: could support configurable binary idx if ever needed.
-		args = args.appendComputeNodeRuntime(v, 0)
+		args = args.appendHostedRuntime(v, v.teeHardware, 0)
 	}
 
 	if err := worker.net.startOasisNode(&worker.Node, nil, args); err != nil {

go/oasis-test-runner/oasis/fixture.go

@@ -199,10 +199,10 @@ type RuntimeFixture struct { // nolint: maligned
 	Entity     int                  `json:"entity"`
 	Keymanager int                  `json:"keymanager"`
 
-	Binaries         []string         `json:"binaries"`
-	GenesisState     storage.WriteLog `json:"genesis_state,omitempty"`
-	GenesisStatePath string           `json:"genesis_state_path,omitempty"`
-	GenesisRound     uint64           `json:"genesis_round,omitempty"`
+	Binaries         map[node.TEEHardware][]string `json:"binaries"`
+	GenesisState     storage.WriteLog              `json:"genesis_state,omitempty"`
+	GenesisStatePath string                        `json:"genesis_state_path,omitempty"`
+	GenesisRound     uint64                        `json:"genesis_round,omitempty"`
 
 	Executor     registry.ExecutorParameters     `json:"executor"`
 	TxnScheduler registry.TxnSchedulerParameters `json:"txn_scheduler"`

go/oasis-test-runner/oasis/keymanager.go

@@ -275,7 +275,7 @@ func (km *Keymanager) startNode() error {
 		runtimeProvisioner(runtimeRegistry.RuntimeProvisionerSandboxed).
 		runtimeSGXLoader(km.net.cfg.RuntimeSGXLoaderBinary).
 		// XXX: could support configurable binary idx if ever needed.
-		runtimePath(km.runtime.id, km.runtime.binaries[0]).
+		runtimePath(km.runtime.id, km.runtime.binaries[km.runtime.teeHardware][0]).
 		workerKeymanagerEnabled().
 		workerKeymanagerRuntimeID(km.runtime.id).
 		configureDebugCrashPoints(km.crashPointsProbability).

go/oasis-test-runner/oasis/runtime.go

@@ -31,7 +31,7 @@ type Runtime struct { // nolint: maligned
 	id   common.Namespace
 	kind registry.RuntimeKind
 
-	binaries    []string
+	binaries    map[node.TEEHardware][]string
 	teeHardware node.TEEHardware
 	mrEnclaves  []*sgx.MrEnclave
 	mrSigner    *sgx.MrSigner
@@ -52,7 +52,7 @@ type RuntimeCfg struct { // nolint: maligned
 	TEEHardware node.TEEHardware
 	MrSigner    *sgx.MrSigner
 
-	Binaries         []string
+	Binaries         map[node.TEEHardware][]string
 	GenesisState     storage.WriteLog
 	GenesisStatePath string
 	GenesisRound     uint64
@@ -162,7 +162,7 @@ func (net *Network) NewRuntime(cfg *RuntimeCfg) (*Runtime, error) {
 	var mrEnclaves []*sgx.MrEnclave
 	if cfg.TEEHardware == node.TEEHardwareIntelSGX {
 		enclaveIdentities := []sgx.EnclaveIdentity{}
-		for _, binary := range cfg.Binaries {
+		for _, binary := range cfg.Binaries[node.TEEHardwareIntelSGX] {
 			var mrEnclave *sgx.MrEnclave
 			if mrEnclave, err = deriveMrEnclave(binary); err != nil {
 				return nil, err

go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
+	"github.com/oasisprotocol/oasis-core/go/common/node"
 	"github.com/oasisprotocol/oasis-core/go/common/sgx"
 	keymanager "github.com/oasisprotocol/oasis-core/go/keymanager/api"
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env"
@@ -46,16 +47,16 @@ func (sc *kmUpgradeImpl) Fixture() (*oasis.NetworkFixture, error) {
 	}
 
 	// Load the upgraded keymanager binary.
-	newKmBinary, err := sc.resolveRuntimeBinary("simple-keymanager-upgrade")
-	if err != nil {
-		return nil, fmt.Errorf("error resolving binary: %w", err)
-	}
+	newKmBinaries := sc.resolveRuntimeBinaries([]string{"simple-keymanager-upgrade"})
 	// Setup the upgraded runtime.
 	kmRuntimeFix := f.Runtimes[0]
 	if kmRuntimeFix.Kind != registry.KindKeyManager {
 		return nil, fmt.Errorf("expected first runtime in fixture to be keymanager runtime, got: %s", kmRuntimeFix.Kind)
 	}
-	kmRuntimeFix.Binaries = append([]string{newKmBinary}, kmRuntimeFix.Binaries...)
+	for _, tee := range []node.TEEHardware{node.TEEHardwareInvalid, node.TEEHardwareIntelSGX} {
+		newKmBinaries[tee] = append(newKmBinaries[tee], kmRuntimeFix.Binaries[tee]...)
+	}
+	kmRuntimeFix.Binaries = newKmBinaries
 	// The upgraded runtime will be registered later.
 	kmRuntimeFix.ExcludeFromGenesis = true
 	f.Runtimes = append(f.Runtimes, kmRuntimeFix)

go/oasis-test-runner/scenario/e2e/runtime/multiple_runtimes.go

@@ -7,6 +7,7 @@ import (
 
 	beacon "github.com/oasisprotocol/oasis-core/go/beacon/api"
 	"github.com/oasisprotocol/oasis-core/go/common"
+	"github.com/oasisprotocol/oasis-core/go/common/node"
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env"
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis"
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/scenario"
@@ -56,13 +57,13 @@ func (sc *multipleRuntimesImpl) Fixture() (*oasis.NetworkFixture, error) {
 	// Remove existing compute runtimes from fixture, remember RuntimeID and
 	// binary from the first one.
 	var id common.Namespace
-	var runtimeBinary string
+	var runtimeBinaries map[node.TEEHardware][]string
 	var rts []oasis.RuntimeFixture
 	for _, rt := range f.Runtimes {
 		if rt.Kind == registry.KindCompute {
-			if runtimeBinary == "" {
+			if runtimeBinaries == nil {
 				copy(id[:], rt.ID[:])
-				runtimeBinary = rt.Binaries[0]
+				runtimeBinaries = rt.Binaries
 			}
 		} else {
 			rts = append(rts, rt)
@@ -84,7 +85,7 @@ func (sc *multipleRuntimesImpl) Fixture() (*oasis.NetworkFixture, error) {
 			Kind:       registry.KindCompute,
 			Entity:     0,
 			Keymanager: 0,
-			Binaries:   []string{runtimeBinary},
+			Binaries:   runtimeBinaries,
 			Executor: registry.ExecutorParameters{
 				GroupSize:       uint64(executorGroupSize),
 				GroupBackupSize: 0,

go/oasis-test-runner/scenario/e2e/runtime/runtime.go

@@ -26,12 +26,13 @@ import (
 )
 
 const (
-	cfgClientBinaryDir  = "client.binary_dir"
-	cfgRuntimeBinaryDir = "runtime.binary_dir"
-	cfgRuntimeLoader    = "runtime.loader"
-	cfgTEEHardware      = "tee_hardware"
-	cfgIasMock          = "ias.mock"
-	cfgEpochInterval    = "epoch.interval"
+	cfgClientBinaryDir          = "client.binary_dir"
+	cfgRuntimeBinaryDirDefault  = "runtime.binary_dir.default"
+	cfgRuntimeBinaryDirIntelSGX = "runtime.binary_dir.intel-sgx"
+	cfgRuntimeLoader            = "runtime.loader"
+	cfgTEEHardware              = "tee_hardware"
+	cfgIasMock                  = "ias.mock"
+	cfgEpochInterval            = "epoch.interval"
 )
 
 var (
@@ -78,7 +79,8 @@ func newRuntimeImpl(name, clientBinary string, clientArgs []string) *runtimeImpl
 		clientArgs:   clientArgs,
 	}
 	sc.Flags.String(cfgClientBinaryDir, "", "path to the client binaries directory")
-	sc.Flags.String(cfgRuntimeBinaryDir, "", "path to the runtime binaries directory")
+	sc.Flags.String(cfgRuntimeBinaryDirDefault, "", "(no-TEE) path to the runtime binaries directory")
+	sc.Flags.String(cfgRuntimeBinaryDirIntelSGX, "", "(Intel SGX) path to the runtime binaries directory")
 	sc.Flags.String(cfgRuntimeLoader, "oasis-core-runtime-loader", "path to the runtime loader")
 	sc.Flags.String(cfgTEEHardware, "", "TEE hardware to use")
 	sc.Flags.Bool(cfgIasMock, true, "if mock IAS service should be used")
@@ -113,14 +115,8 @@ func (sc *runtimeImpl) Fixture() (*oasis.NetworkFixture, error) {
 	if tee == node.TEEHardwareIntelSGX {
 		mrSigner = &sgx.FortanixDummyMrSigner
 	}
-	keyManagerBinary, err := sc.resolveDefaultKeyManagerBinary()
-	if err != nil {
-		return nil, err
-	}
-	runtimeBinary, err := sc.resolveRuntimeBinary("simple-keyvalue")
-	if err != nil {
-		return nil, err
-	}
+	keyManagerBinary := "simple-keymanager"
+	runtimeBinary := "simple-keyvalue"
 	runtimeLoader, _ := sc.Flags.GetString(cfgRuntimeLoader)
 	iasMock, _ := sc.Flags.GetBool(cfgIasMock)
 	ff := &oasis.NetworkFixture{
@@ -151,15 +147,15 @@ func (sc *runtimeImpl) Fixture() (*oasis.NetworkFixture, error) {
 				AdmissionPolicy: registry.RuntimeAdmissionPolicy{
 					AnyNode: &registry.AnyNodeRuntimeAdmissionPolicy{},
 				},
-				Binaries: []string{keyManagerBinary},
+				Binaries: sc.resolveRuntimeBinaries([]string{keyManagerBinary}),
 			},
 			// Compute runtime.
 			{
 				ID:         runtimeID,
 				Kind:       registry.KindCompute,
 				Entity:     0,
 				Keymanager: 0,
-				Binaries:   []string{runtimeBinary},
+				Binaries:   sc.resolveRuntimeBinaries([]string{runtimeBinary}),
 				Executor: registry.ExecutorParameters{
 					GroupSize:       2,
 					GroupBackupSize: 1,
@@ -254,26 +250,31 @@ func (sc *runtimeImpl) resolveClientBinary(clientBinary string) string {
 	return filepath.Join(cbDir, clientBinary)
 }
 
-func (sc *runtimeImpl) resolveRuntimeBinary(runtimeBinary string) (string, error) {
-	tee, err := sc.getTEEHardware()
-	if err != nil {
-		return "", err
+func (sc *runtimeImpl) resolveRuntimeBinaries(runtimeBinaries []string) map[node.TEEHardware][]string {
+	binaries := make(map[node.TEEHardware][]string)
+	for _, tee := range []node.TEEHardware{
+		node.TEEHardwareInvalid,
+		node.TEEHardwareIntelSGX,
+	} {
+		for _, binary := range runtimeBinaries {
+			binaries[tee] = append(binaries[tee], sc.resolveRuntimeBinary(binary, tee))
+		}
 	}
+	return binaries
+}
 
-	var runtimeExt string
+func (sc *runtimeImpl) resolveRuntimeBinary(runtimeBinary string, tee node.TEEHardware) string {
+	var runtimeExt, path string
 	switch tee {
 	case node.TEEHardwareInvalid:
 		runtimeExt = ""
+		path, _ = sc.Flags.GetString(cfgRuntimeBinaryDirDefault)
 	case node.TEEHardwareIntelSGX:
 		runtimeExt = ".sgxs"
+		path, _ = sc.Flags.GetString(cfgRuntimeBinaryDirIntelSGX)
 	}
 
-	rtBinDir, _ := sc.Flags.GetString(cfgRuntimeBinaryDir)
-	return filepath.Join(rtBinDir, runtimeBinary+runtimeExt), nil
-}
-
-func (sc *runtimeImpl) resolveDefaultKeyManagerBinary() (string, error) {
-	return sc.resolveRuntimeBinary("simple-keymanager")
+	return filepath.Join(path, runtimeBinary+runtimeExt)
 }
 
 func (sc *runtimeImpl) startClient(childEnv *env.Env) (*exec.Cmd, error) {

go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
+	"github.com/oasisprotocol/oasis-core/go/common/node"
 	"github.com/oasisprotocol/oasis-core/go/common/sgx"
 	keymanager "github.com/oasisprotocol/oasis-core/go/keymanager/api"
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env"
@@ -56,14 +57,14 @@ func (sc *runtimeUpgradeImpl) Fixture() (*oasis.NetworkFixture, error) {
 	}
 
 	// Load the upgraded runtime binary.
-	newRuntimeBinary, err := sc.resolveRuntimeBinary("simple-keyvalue-upgrade")
-	if err != nil {
-		return nil, fmt.Errorf("error resolving upgraded binary: %w", err)
-	}
+	newRuntimeBinaries := sc.resolveRuntimeBinaries([]string{"simple-keyvalue-upgrade"})
 
 	// Setup the upgraded runtime (first is keymanager, others should be generic compute).
 	runtimeFix := f.Runtimes[computeIndex]
-	runtimeFix.Binaries = append([]string{newRuntimeBinary}, runtimeFix.Binaries...)
+	for _, tee := range []node.TEEHardware{node.TEEHardwareInvalid, node.TEEHardwareIntelSGX} {
+		newRuntimeBinaries[tee] = append(newRuntimeBinaries[tee], runtimeFix.Binaries[tee]...)
+	}
+	runtimeFix.Binaries = newRuntimeBinaries
 
 	// The upgraded runtime will be registered later.
 	runtimeFix.ExcludeFromGenesis = true

go/runtime/client/api/api.go

@@ -29,9 +29,11 @@ var (
 	ErrInternal = errors.New(ModuleName, 2, "client: internal error")
 	// ErrTransactionExpired is an error returned when transaction expired.
 	ErrTransactionExpired = errors.New(ModuleName, 3, "client: transaction expired")
-	// ErrNotSynced is an error return if transaction is submitted before node has finished
+	// ErrNotSynced is an error returned if transaction is submitted before node has finished
 	// initial syncing.
 	ErrNotSynced = errors.New(ModuleName, 4, "client: not finished initial sync")
+	// ErrCheckTxFailed is an error returned if the local transaction check fails.
+	ErrCheckTxFailed = errors.New(ModuleName, 5, "client: transaction check failed")
 )
 
 // RuntimeClient is the runtime client interface.