go/e2e/txsource: periodically restart runtime nodes

.changelog/3124.intenral.md

@@ -0,0 +1 @@
+go/e2e/txsource: Periodically restart runtime nodes

go/oasis-node/cmd/debug/txsource/workload/runtime.go

@@ -6,6 +6,7 @@ import (
 	"math/rand"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	flag "github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"google.golang.org/grpc"
@@ -112,15 +113,22 @@ func (r *runtime) submitRuntimeRquest(ctx context.Context, rtc runtimeClient.Run
 		"request", req,
 	)
 
-	// Wait for a maximum of 30 seconds as invalid submissions may block
-	// forever.
-	submitCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
-	out, err := rtc.SubmitTx(submitCtx, rtx)
+	// Retry in case a runtime node is just getting restarted.
+	sched := backoff.NewExponentialBackOff()
+	sched.MaxInterval = 120
+	sched.MaxElapsedTime = 10
+
+	var out []byte
+	err := backoff.Retry(func() (err error) {
+		out, err = rtc.SubmitTx(ctx, rtx)
+		r.logger.Error("submitting runtime request error",
+			"err", err,
+		)
+		return
+	}, backoff.WithContext(sched, ctx))
 	if err != nil {
-		cancel()
 		return nil, fmt.Errorf("failed to submit runtime transaction: %w", err)
 	}
-	cancel()
 
 	if err = cbor.Unmarshal(out, &rsp); err != nil {
 		return nil, fmt.Errorf("malformed tx output from runtime: %w", err)

go/oasis-test-runner/scenario/e2e/runtime/txsource.go

@@ -50,7 +50,7 @@ var TxSourceMultiShort scenario.Scenario = &txSourceImpl{
 		workload.NameTransfer,
 	},
 	timeLimit:                         timeLimitShort,
-	livenessCheckInterval:             livenessCheckInterval,
+	nodeRestartInterval:               nodeRestartIntervalLong,
 	consensusPruneDisabledProbability: 0.1,
 	consensusPruneMinKept:             100,
 	consensusPruneMaxKept:             200,
@@ -148,11 +148,10 @@ func (sc *txSourceImpl) Fixture() (*oasis.NetworkFixture, error) {
 	f.Network.StakingGenesis = "tests/fixture-data/txsource/staking-genesis.json"
 
 	if sc.nodeRestartInterval > 0 {
-		// If node restarts enabled, do not enable round timeouts and merge
+		// If node restarts enabled, do not enable round timeouts and
 		// discrepancy log watchers.
 		f.Network.DefaultLogWatcherHandlerFactories = []log.WatcherHandlerFactory{
 			oasis.LogAssertNoRoundFailures(),
-			oasis.LogAssertNoExecutionDiscrepancyDetected(),
 		}
 	}
 
@@ -167,6 +166,16 @@ func (sc *txSourceImpl) Fixture() (*oasis.NetworkFixture, error) {
 		{Entity: 1},
 		{Entity: 1},
 	}
+	f.ComputeWorkers = []oasis.ComputeWorkerFixture{
+		{Entity: 1},
+		{Entity: 1},
+		{Entity: 1},
+		{Entity: 1},
+	}
+	f.Keymanagers = []oasis.KeymanagerFixture{
+		{Runtime: 0, Entity: 1},
+		{Runtime: 0, Entity: 1},
+	}
 
 	// Update validators to require fee payments.
 	for i := range f.Validators {
@@ -211,10 +220,19 @@ func (sc *txSourceImpl) manager(env *env.Env, errCh chan error) {
 
 	// Randomize node order.
 	var nodes []*oasis.Node
-	for _, v := range sc.Net.Validators() {
+	// Keep one of each types of nodes always running.
+	for _, v := range sc.Net.Validators()[1:] {
 		nodes = append(nodes, &v.Node)
 	}
-	// TODO: Consider including storage/compute workers.
+	for _, s := range sc.Net.StorageWorkers()[1:] {
+		nodes = append(nodes, &s.Node)
+	}
+	for _, c := range sc.Net.ComputeWorkers()[1:] {
+		nodes = append(nodes, &c.Node)
+	}
+	for _, k := range sc.Net.Keymanagers()[1:] {
+		nodes = append(nodes, &k.Node)
+	}
 
 	restartTicker := time.NewTicker(sc.nodeRestartInterval)
 	defer restartTicker.Stop()

tests/fixture-data/txsource/staking-genesis.json

@@ -17,7 +17,7 @@
       "transfer": 10
     }
   },
-  "total_supply": "100000000000",
+  "total_supply": "120000000000",
   "ledger": {
     "oasis1qpt202cf6t0s5ugkk34p83yf0c30gpjkny92u7dh": {"general": {"balance": "10000000000"}},
     "oasis1qryg8qf3ydzcphr328l8psz007fms9dxeuy8lgzq": {"general": {"balance": "10000000000"}},
@@ -27,10 +27,12 @@
     "oasis1qp6tl30ljsrrqnw2awxxu2mtxk0qxyy2nymtsy90": {"general": {"balance": "10000000000"}},
     "oasis1qr77y0cqdzcqgz2wqkv59yz0j4vfvyryfv8vxllt": {"general": {"balance": "10000000000"}},
     "oasis1qzyw75ds6nw0af98xfmmpl3z8sgf3mdslvtzzcn6": {"general": {"balance": "10000000000"}},
+    "oasis1qrp7l53vn6h2z7p242ldtkqtttz2jf9dwsgu05aa": {"general": {"balance": "10000000000"}},
 
     "oasis1qrhp36j49ncpaac0aufwyuvtk04nfxcj2yq7y4my": {"general": {"balance": "10000000000"}},
     "oasis1qzc2fexm30puzq2cmlm832fvpnyaxrq33cx4zukj": {"general": {"balance": "10000000000"}},
 
-    "oasis1qpx0k28va6n0r25qd2j4jdh9f42n5vex6s9lp780": {"general": {"balance": "10000000000"}}
+    "oasis1qpx0k28va6n0r25qd2j4jdh9f42n5vex6s9lp780": {"general": {"balance": "10000000000"}},
+    "oasis1qz30d8mqzrsrsu7fr0e6nxk0ze7ffdkj8ur7sqp0": {"general": {"balance": "10000000000"}}
   }
 }