go/oasis-test-runner: configurable sanitychecker

.changelog/3719.internal.1.md

@@ -0,0 +1,4 @@
+go/oasis-test-runner: Configurable supplementary sanity checker
+
+Before the supplementary sanity checker was always automatically started on
+the validator-0.

.changelog/3719.internal.2.md

@@ -0,0 +1,3 @@
+go/oasis-test-runner: Add support for configuring `pprof`
+
+Adds support for configuring `pprof` on the test nodes.

go/oasis-net-runner/fixtures/default.go

@@ -78,7 +78,7 @@ func newDefaultFixture() (*oasis.NetworkFixture, error) {
 			{IsDebugTestEntity: true},
 		},
 		Validators: []oasis.ValidatorFixture{
-			{Entity: 1},
+			{Entity: 1, Consensus: oasis.ConsensusFixture{SupplementarySanityInterval: 1}},
 		},
 		Seeds: []oasis.SeedFixture{{}},
 	}

go/oasis-node/cmd/common/pprof/pprof.go

@@ -18,7 +18,8 @@ import (
 	"github.com/oasisprotocol/oasis-core/go/common/service"
 )
 
-const cfgPprofBind = "pprof.bind"
+// CfgPprofBind enables profiling endpoint at the given address.
+const CfgPprofBind = "pprof.bind"
 
 // Flags has the flags used by the pprof service.
 var Flags = flag.NewFlagSet("", flag.ContinueOnError)
@@ -118,7 +119,7 @@ func (p *pprofService) Cleanup() {
 
 // New constructs a new pprof service.
 func New(ctx context.Context) (service.BackgroundService, error) {
-	address := viper.GetString(cfgPprofBind)
+	address := viper.GetString(CfgPprofBind)
 
 	return &pprofService{
 		BaseBackgroundService: *service.NewBaseBackgroundService("pprof"),
@@ -129,7 +130,7 @@ func New(ctx context.Context) (service.BackgroundService, error) {
 }
 
 func init() {
-	Flags.String(cfgPprofBind, "", "enable profiling endpoint at given address")
+	Flags.String(CfgPprofBind, "", "enable profiling endpoint at given address")
 
 	_ = viper.BindPFlags(Flags)
 }

go/oasis-node/cmd/debug/txsource/workload/governance.go

@@ -25,28 +25,31 @@ import (
 // NameGovernance is the name of the governance workload.
 const NameGovernance = "governance"
 
-var errUnexpectedGovTxResult = fmt.Errorf("unexpected governance tx result")
+var (
 
-// Governance is the governance workload.
-var Governance = &governanceWorkload{
-	BaseWorkload: NewBaseWorkload(NameGovernance),
-}
+	// Governance is the governance workload.
+	Governance = &governanceWorkload{
+		BaseWorkload: NewBaseWorkload(NameGovernance),
+	}
 
-var numProposerAccounts = 10
-
-// How likely voters should vote YES for the proposal made by i'th proposer.
-var proposersVoteYesRate = []uint8{
-	100,
-	99,
-	98,
-	95,
-	92,
-	90,
-	80,
-	70,
-	50,
-	0,
-}
+	// Timeout after each governance workload iteration.
+	iterationTimeout         = 2 * time.Second
+	errUnexpectedGovTxResult = fmt.Errorf("unexpected governance tx result")
+	numProposerAccounts      = 10
+	// How likely voters should vote YES for the proposal made by i'th proposer.
+	proposersVoteYesRate = []uint8{
+		100,
+		99,
+		98,
+		95,
+		92,
+		90,
+		80,
+		70,
+		50,
+		0,
+	}
+)
 
 type governanceWorkload struct {
 	BaseWorkload
@@ -247,6 +250,7 @@ OUTER:
 	if pendingUpgrade == nil {
 		g.Logger.Debug("no eligible pending upgrade for submitting cancel upgrade proposal, skipping",
 			"pending_upgrades", pendingUpgrades,
+			"current_epoch", g.currentEpoch,
 		)
 		return nil
 	}
@@ -279,7 +283,11 @@ func (g *governanceWorkload) submitVote(voter signature.Signer, proposalID uint6
 		return nil
 	case errors.Is(err, registry.ErrNoSuchNode),
 		errors.Is(err, governance.ErrNotEligible):
-		g.Logger.Error("submitting vote error: voter not a validator, continuing", "err", err)
+		g.Logger.Error("submitting vote error: voter not a validator, continuing",
+			"err", err,
+			"voter", voter.Public(),
+			"proposal_id", proposalID,
+		)
 		return nil
 	default:
 		return fmt.Errorf("failed to sign and submit cast vote transaction: %w", err)
@@ -411,7 +419,7 @@ func (g *governanceWorkload) Run(
 	// Main workload loop.
 	for {
 		select {
-		case <-time.After(1 * time.Second):
+		case <-time.After(iterationTimeout):
 		case <-gracefulExit.Done():
 			g.Logger.Debug("time's up")
 			return nil
@@ -433,12 +441,15 @@ func (g *governanceWorkload) Run(
 			var upgrades []*upgrade.Descriptor
 			upgrades, err = g.governance.PendingUpgrades(g.ctx, consensus.HeightLatest)
 			if err != nil {
-				return fmt.Errorf("querying penging upgrades: %w", err)
+				return fmt.Errorf("querying pending upgrades: %w", err)
 			}
 			for _, up := range upgrades {
 				if up.Epoch.AbsDiff(g.currentEpoch) != g.parameters.UpgradeCancelMinEpochDiff+2 {
 					continue
 				}
+				g.Logger.Debug("ensuring pending upgrade canceled",
+					"upgrade", up,
+				)
 				if err = g.ensureUpgradeCanceled(up); err != nil {
 					return fmt.Errorf("ensuring upgrade canceled: %w", err)
 				}

go/oasis-node/cmd/debug/txsource/workload/queries.go

@@ -68,6 +68,10 @@ const (
 	// nodes have gap in history for the queried height, due to restoring from checkpoint. See also:
 	// https://github.com/oasisprotocol/oasis-core/issues/3337
 	queriesNumAllowedQueryTxsHistoricalFailures = 5
+
+	// doQueryAllProposalsEvery configures how often the queries workload should query all (including)
+	// past proposals.
+	doQueryAllProposalsEvery = 10
 )
 
 // QueriesFlags are the queries workload flags.
@@ -94,6 +98,8 @@ type queries struct {
 	queryTxsHistoricalFailures uint64
 
 	runtimeGenesisRound uint64
+
+	iteration uint64
 }
 
 func (q *queries) sanityCheckTransactionEvents(ctx context.Context, height int64, txEvents []*results.Event) error {
@@ -602,11 +608,34 @@ func (q *queries) doGovernanceQueries(ctx context.Context, rng *rand.Rand, heigh
 	q.logger.Debug("doing governance queries",
 		"height", height,
 	)
-	proposals, err := q.governance.Proposals(ctx, height)
+
+	if q.iteration%doQueryAllProposalsEvery == 0 {
+		proposals, err := q.governance.Proposals(ctx, height)
+		if err != nil {
+			return fmt.Errorf("governance.Proposals: %w", err)
+		}
+		for _, p := range proposals {
+			var p2 *governance.Proposal
+			p2, err = q.governance.Proposal(ctx, &governance.ProposalQuery{Height: height, ProposalID: p.ID})
+			if err != nil {
+				return fmt.Errorf("governance.Proposal: %w", err)
+			}
+			if !p.Content.Equals(&p2.Content) {
+				return fmt.Errorf("proposal contents not equal")
+			}
+
+			_, err = q.governance.Votes(ctx, &governance.ProposalQuery{Height: height, ProposalID: p.ID})
+			if err != nil {
+				return fmt.Errorf("governance.Votes: %w", err)
+			}
+		}
+	}
+
+	activeProposals, err := q.governance.ActiveProposals(ctx, height)
 	if err != nil {
-		return fmt.Errorf("governance.Proposals: %w", err)
+		return fmt.Errorf("governance.ActiveProposals: %w", err)
 	}
-	for _, p := range proposals {
+	for _, p := range activeProposals {
 		var p2 *governance.Proposal
 		p2, err = q.governance.Proposal(ctx, &governance.ProposalQuery{Height: height, ProposalID: p.ID})
 		if err != nil {
@@ -622,14 +651,6 @@ func (q *queries) doGovernanceQueries(ctx context.Context, rng *rand.Rand, heigh
 		}
 	}
 
-	activeProposals, err := q.governance.ActiveProposals(ctx, height)
-	if err != nil {
-		return fmt.Errorf("governance.ActiveProposals: %w", err)
-	}
-	if l1, l2 := len(activeProposals), len(proposals); l1 > l2 {
-		return fmt.Errorf("more active proposals(%v) than all proposals(%v)", l1, l2)
-	}
-
 	pendingUpgrades, err := q.governance.PendingUpgrades(ctx, height)
 	if err != nil {
 		return fmt.Errorf("governance.PendingUpgrades: %w", err)
@@ -916,6 +937,7 @@ func (q *queries) Run(
 		default:
 			return err
 		}
+		q.iteration++
 
 		select {
 		case <-time.After(1 * time.Second):

go/oasis-test-runner/oasis/args.go

@@ -26,6 +26,7 @@ import (
 	"github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/flags"
 	"github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/grpc"
 	"github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/metrics"
+	"github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/common/pprof"
 	"github.com/oasisprotocol/oasis-core/go/oasis-node/cmd/debug/byzantine"
 	runtimeClient "github.com/oasisprotocol/oasis-core/go/runtime/client"
 	runtimeRegistry "github.com/oasisprotocol/oasis-core/go/runtime/registry"
@@ -66,6 +67,16 @@ func (args *argBuilder) debugAllowTestKeys() *argBuilder {
 	return args
 }
 
+func (args *argBuilder) debugEnableProfiling(port uint16) *argBuilder {
+	if port == 0 {
+		return args
+	}
+	args.vec = append(args.vec,
+		"--"+pprof.CfgPprofBind, "0.0.0.0:"+strconv.Itoa(int(port)),
+	)
+	return args
+}
+
 func (args *argBuilder) grpcServerPort(port uint16) *argBuilder {
 	args.vec = append(args.vec, []string{
 		"--" + grpc.CfgServerPort, strconv.Itoa(int(port)),
@@ -208,11 +219,13 @@ func (args *argBuilder) runtimeSupported(id common.Namespace) *argBuilder {
 	return args
 }
 
-func (args *argBuilder) tendermintSupplementarySanityEnabled() *argBuilder {
-	args.vec = append(args.vec, "--"+tendermintFull.CfgSupplementarySanityEnabled)
-	args.vec = append(args.vec, []string{
-		"--" + tendermintFull.CfgSupplementarySanityInterval, "1",
-	}...)
+func (args *argBuilder) tendermintSupplementarySanity(interval uint64) *argBuilder {
+	if interval > 0 {
+		args.vec = append(args.vec, "--"+tendermintFull.CfgSupplementarySanityEnabled)
+		args.vec = append(args.vec, []string{
+			"--" + tendermintFull.CfgSupplementarySanityInterval, strconv.Itoa(int(interval)),
+		}...)
+	}
 	return args
 }
 

go/oasis-test-runner/oasis/byzantine.go

@@ -41,6 +41,7 @@ func (worker *Byzantine) startNode() error {
 	args := newArgBuilder().
 		debugDontBlameOasis().
 		debugAllowTestKeys().
+		debugEnableProfiling(worker.Node.pprofPort).
 		tendermintDebugAllowDuplicateIP().
 		tendermintCoreAddress(worker.consensusPort).
 		tendermintDebugAddrBookLenient().
@@ -118,11 +119,16 @@ func (net *Network) NewByzantine(cfg *ByzantineCfg) (*Byzantine, error) {
 		activationEpoch: cfg.ActivationEpoch,
 		runtime:         cfg.Runtime,
 	}
+	net.nextNodePort += 2
 	worker.doStartNode = worker.startNode
 	copy(worker.NodeID[:], nodeKey[:])
 
+	if cfg.EnableProfiling {
+		worker.Node.pprofPort = net.nextNodePort
+		net.nextNodePort++
+	}
+
 	net.byzantine = append(net.byzantine, worker)
-	net.nextNodePort += 2
 
 	if err := net.AddLogWatcher(&worker.Node); err != nil {
 		net.logger.Error("failed to add log watcher",

go/oasis-test-runner/oasis/client.go

@@ -4,13 +4,13 @@ import (
 	"fmt"
 
 	"github.com/oasisprotocol/oasis-core/go/common/node"
-	registry "github.com/oasisprotocol/oasis-core/go/registry/api"
 )
 
 // Client is an Oasis client node.
 type Client struct {
 	Node
 
+	runtimes          []int
 	maxTransactionAge int64
 
 	consensusPort uint16
@@ -21,30 +21,33 @@ type Client struct {
 type ClientCfg struct {
 	NodeCfg
 
+	Runtimes          []int
 	MaxTransactionAge int64
 }
 
 func (client *Client) startNode() error {
 	args := newArgBuilder().
 		debugDontBlameOasis().
 		debugAllowTestKeys().
+		debugEnableProfiling(client.Node.pprofPort).
 		tendermintPrune(client.consensus.PruneNumKept).
 		tendermintRecoverCorruptedWAL(client.consensus.TendermintRecoverCorruptedWAL).
 		tendermintCoreAddress(client.consensusPort).
 		appendNetwork(client.net).
 		appendSeedNodes(client.net.seeds).
 		workerP2pPort(client.p2pPort).
 		workerP2pEnabled().
-		runtimeTagIndexerBackend("bleve")
+		tendermintSupplementarySanity(client.supplementarySanityInterval)
 
 	if client.maxTransactionAge != 0 {
 		args = args.runtimeClientMaxTransactionAge(client.maxTransactionAge)
 	}
 
-	for _, v := range client.net.runtimes {
-		if v.kind != registry.KindCompute {
-			continue
-		}
+	if len(client.runtimes) > 0 {
+		args = args.runtimeTagIndexerBackend("bleve")
+	}
+	for _, idx := range client.runtimes {
+		v := client.net.runtimes[idx]
 		// XXX: could support configurable binary idx if ever needed.
 		args = args.appendHostedRuntime(v, node.TEEHardwareInvalid, 0)
 	}
@@ -76,21 +79,29 @@ func (net *Network) NewClient(cfg *ClientCfg) (*Client, error) {
 
 	client := &Client{
 		Node: Node{
-			Name:        clientName,
-			net:         net,
-			dir:         clientDir,
-			consensus:   cfg.Consensus,
-			termEarlyOk: cfg.AllowEarlyTermination,
-			termErrorOk: cfg.AllowErrorTermination,
+			Name:                        clientName,
+			net:                         net,
+			dir:                         clientDir,
+			consensus:                   cfg.Consensus,
+			termEarlyOk:                 cfg.AllowEarlyTermination,
+			termErrorOk:                 cfg.AllowErrorTermination,
+			supplementarySanityInterval: cfg.SupplementarySanityInterval,
 		},
+		runtimes:          cfg.Runtimes,
 		maxTransactionAge: cfg.MaxTransactionAge,
 		consensusPort:     net.nextNodePort,
 		p2pPort:           net.nextNodePort + 1,
 	}
+	net.nextNodePort += 2
+
 	client.doStartNode = client.startNode
 
+	if cfg.EnableProfiling {
+		client.Node.pprofPort = net.nextNodePort
+		net.nextNodePort++
+	}
+
 	net.clients = append(net.clients, client)
-	net.nextNodePort += 2
 
 	return client, nil
 }