go/roothash: Slash runtime compute nodes for incorrect results

go/consensus/tendermint/apps/roothash/roothash.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
+	"github.com/oasisprotocol/oasis-core/go/common/crypto/signature"
 	"github.com/oasisprotocol/oasis-core/go/common/logging"
 	"github.com/oasisprotocol/oasis-core/go/consensus/api/transaction"
 	tmapi "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/api"
@@ -522,6 +523,58 @@ func (app *rootHashApplication) tryFinalizeExecutorCommits(
 			return fmt.Errorf("failed to process runtime messages: %w", err)
 		}
 
+		// If there was a discrepancy, slash nodes for incorrect results if configured.
+		if rtState.ExecutorPool.Discrepancy {
+			ctx.Logger().Debug("executor pool discrepancy",
+				"slashing", runtime.Staking.Slashing,
+			)
+			if penalty, ok := rtState.Runtime.Staking.Slashing[staking.SlashRuntimeIncorrectResults]; ok && !penalty.Amount.IsZero() {
+				commitments := rtState.ExecutorPool.ExecuteCommitments
+				var (
+					// Worker nodes that submitted incorrect results get slashed.
+					workersIncorrectResults []signature.PublicKey
+					// Backup worker nodes that resolved the discrepancy get rewarded.
+					backupCorrectResults []signature.PublicKey
+				)
+				for _, n := range rtState.ExecutorPool.Committee.Members {
+					c, ok := commitments[n.PublicKey]
+					if !ok {
+						continue
+					}
+					switch {
+					case n.Role == scheduler.RoleBackupWorker:
+						// Backup workers that resolved the commitment.
+						if !commit.MostlyEqual(c) {
+							continue
+						}
+						ctx.Logger().Debug("backup worker resolved the commitment",
+							"pubkey", n.PublicKey,
+						)
+						backupCorrectResults = append(backupCorrectResults, n.PublicKey)
+					case n.Role == scheduler.RoleWorker:
+						// Workers that caused the discrepancy.
+						if commit.MostlyEqual(c) {
+							continue
+						}
+						ctx.Logger().Debug("worker caused the discrepancy",
+							"pubkey", n.PublicKey,
+						)
+						workersIncorrectResults = append(workersIncorrectResults, n.PublicKey)
+					}
+				}
+				// Slash for incorrect results.
+				if err = onRuntimeIncorrectResults(
+					ctx,
+					workersIncorrectResults,
+					backupCorrectResults,
+					runtime.ID,
+					&penalty.Amount,
+				); err != nil {
+					return fmt.Errorf("failed to slash for incorrect results: %w", err)
+				}
+			}
+		}
+
 		// Generate the final block.
 		blk := block.NewEmptyBlock(rtState.CurrentBlock, uint64(ctx.Now().Unix()), block.Normal)
 		blk.Header.IORoot = *hdr.IORoot

go/consensus/tendermint/apps/roothash/slashing.go

@@ -44,11 +44,121 @@ func onEvidenceRuntimeEquivocation(
 	}
 
 	// Move slashed amount to the runtime account.
-	// TODO: part of slashed amount (configurable) should be transferred to the transaction submitter.
+	// TODO: part of slashed amount (configurable) should be transferred to the transaction submitter (should be escrowed?).
 	runtimeAddr := staking.NewRuntimeAddress(runtimeID)
 	if _, err := stakeState.TransferFromCommon(ctx, runtimeAddr, totalSlashed); err != nil {
 		return fmt.Errorf("tendermint/roothash: failed transferring reward to runtime account %s: %w", runtimeAddr, err)
 	}
 
 	return nil
 }
+
+func onRuntimeIncorrectResults(
+	ctx *abciAPI.Context,
+	discrepancyCausers []signature.PublicKey,
+	discrepancyResolvers []signature.PublicKey,
+	runtimeID common.Namespace,
+	penaltyAmount *quantity.Quantity,
+) error {
+	regState := registryState.NewMutableState(ctx.State())
+	stakeState := stakingState.NewMutableState(ctx.State())
+
+	var totalSlashed quantity.Quantity
+	for _, pk := range discrepancyCausers {
+		node, err := regState.Node(ctx, pk)
+		if err != nil {
+			ctx.Logger().Error("failed to get runtime node by commitment signature public key",
+				"public_key", pk,
+				"err", err,
+			)
+			// Don't fail if for whatever reason node cannot be found.
+			continue
+		}
+		entityAddr := staking.NewAddress(node.EntityID)
+		slashed, err := stakeState.SlashEscrow(ctx, entityAddr, penaltyAmount)
+		if err != nil {
+			return fmt.Errorf("tendermint/roothash: error slashing account %s: %w", entityAddr, err)
+		}
+		if err = totalSlashed.Add(slashed); err != nil {
+			return fmt.Errorf("tendermint/roothash: totalSlashed.Add(slashed): %w", err)
+		}
+		ctx.Logger().Debug("runtime node entity slashed for incorrect results",
+			"slashed", slashed,
+			"total_slashed", totalSlashed,
+			"addr", entityAddr,
+		)
+	}
+
+	// It can happen that nothing was slashed as nodes could be out of stake.
+	// This can happen since stake is only checked on epoch transitions. This should
+	// not fail the round/slashing as otherwise a single node without stake could
+	// cause round failures until the epoch transition.
+	if totalSlashed.IsZero() {
+		// Nothing more to do in this case.
+		return nil
+	}
+
+	// Distribute slashed funds.
+	// Runtime account reward.
+	// TODO: make portion that is transferred to the runtime account configurable.
+	runtimePercentage := 50
+	runtimeAccReward := totalSlashed.Clone()
+	if err := runtimeAccReward.Mul(quantity.NewFromUint64(uint64(runtimePercentage))); err != nil {
+		return fmt.Errorf("tendermint/roothash: runtimeAccReward.Mul: %w", err)
+	}
+	if err := runtimeAccReward.Quo(quantity.NewFromUint64(uint64(100))); err != nil {
+		return fmt.Errorf("tendermint/roothash: runtimeAccReward.Quo(100): %w", err)
+	}
+	runtimeAddr := staking.NewRuntimeAddress(runtimeID)
+	if _, err := stakeState.TransferFromCommon(ctx, runtimeAddr, runtimeAccReward); err != nil {
+		return fmt.Errorf("tendermint/roothash: failed transferring reward to %s: %w", runtimeAddr, err)
+	}
+	ctx.Logger().Debug("runtime account awarded slashed funds",
+		"reward", runtimeAccReward,
+		"total_slashed", totalSlashed,
+		"runtime_addr", runtimeAddr,
+	)
+
+	// Backup workers reward.
+	// TODO: should probably be escrowed.
+	var rewardEntities []signature.PublicKey
+	for _, pk := range discrepancyResolvers {
+		node, err := regState.Node(ctx, pk)
+		if err != nil {
+			ctx.Logger().Error("failed to get runtime node by commitment signature public key",
+				"public_key", pk,
+				"err", err,
+			)
+			// Don't fail if node for whatever reason couldn't be found?
+		}
+		rewardEntities = append(rewardEntities, node.EntityID)
+	}
+	if len(rewardEntities) == 0 {
+		// Nothing more to do.
+		return nil
+	}
+
+	// (totalSlashed - runtimeAccReward) / n_reward_entities
+	entityReward := totalSlashed.Clone()
+	if err := entityReward.Sub(runtimeAccReward); err != nil {
+		return fmt.Errorf("tendermint/roothash: remainingReward.Sub(runtimeAccReward): %w", err)
+	}
+	if err := entityReward.Quo(quantity.NewFromUint64(uint64(len(rewardEntities)))); err != nil {
+		return fmt.Errorf("tendermint/roothash: remainingReward.Quo(len(discrepancyResolvers)): %w", err)
+	}
+
+	for _, pk := range rewardEntities {
+		entAddr := staking.NewAddress(pk)
+		// TODO: should be escrowed.
+		if _, err := stakeState.TransferFromCommon(ctx, entAddr, entityReward); err != nil {
+			return fmt.Errorf("tendermint/roothash: failed transferring reward to %s: %w", entAddr, err)
+		}
+		ctx.Logger().Debug("entity account awarded slashed funds",
+			"reward", entityReward,
+			"total_slashed", totalSlashed,
+			"entity_addr", entAddr,
+		)
+	}
+
+	return nil
+}

go/consensus/tendermint/apps/roothash/slashing_test.go

@@ -1,6 +1,7 @@
 package roothash
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -108,3 +109,142 @@ func TestOnEvidenceRuntimeEquivocation(t *testing.T) {
 	require.NoError(err, "runtime Addr")
 	require.EqualValues(amount, &rtAcc.General.Balance, "runtime account should get slashed amount")
 }
+
+func TestOnRuntimeIncorrectResults(t *testing.T) {
+	require := require.New(t)
+
+	amount := quantity.NewFromUint64(100)
+
+	now := time.Unix(1580461674, 0)
+	appState := abciAPI.NewMockApplicationState(&abciAPI.MockApplicationStateConfig{})
+	ctx := appState.NewContext(abciAPI.ContextBeginBlock, now)
+	defer ctx.Close()
+
+	regState := registryState.NewMutableState(ctx.State())
+	stakeState := stakingState.NewMutableState(ctx.State())
+
+	runtime := registry.Runtime{}
+	missingNodeSigner := memorySigner.NewTestSigner("TestOnRuntimeIncorrectResults missing node signer")
+
+	// Empty lists.
+	err := onRuntimeIncorrectResults(
+		ctx,
+		[]signature.PublicKey{},
+		[]signature.PublicKey{},
+		runtime.ID,
+		amount,
+	)
+	require.NoError(err, "should not fail when there's no nodes to be slashed")
+
+	// Signer not known.
+	err = onRuntimeIncorrectResults(
+		ctx,
+		[]signature.PublicKey{missingNodeSigner.Public()},
+		[]signature.PublicKey{},
+		runtime.ID,
+		amount,
+	)
+	require.NoError(err, "should not fail when node to be slashed cannot be found")
+
+	// Add state.
+	const numNodes, numSlashed = 20, 13
+	var testNodes []*node.Node
+	for i := 0; i < numNodes; i++ {
+		// Add entity.
+		entitySigner := memorySigner.NewTestSigner(fmt.Sprintf("TestOnRuntimeIncorrectResults entity signer: %d", i))
+		ent := &entity.Entity{
+			ID: entitySigner.Public(),
+		}
+		var sigEntity *entity.SignedEntity
+		sigEntity, err = entity.SignEntity(entitySigner, registry.RegisterEntitySignatureContext, ent)
+		require.NoError(err, "SignEntity")
+		err = regState.SetEntity(ctx, ent, sigEntity)
+		require.NoError(err, "SetEntity")
+		// Add node.
+		nodeSigner := memorySigner.NewTestSigner(fmt.Sprintf("TestOnRuntimeIncorrectResults node signer: %d", i))
+		nod := &node.Node{
+			Versioned: cbor.NewVersioned(node.LatestNodeDescriptorVersion),
+			ID:        nodeSigner.Public(),
+			EntityID:  ent.ID,
+			Consensus: node.ConsensusInfo{},
+		}
+		var sigNode *node.MultiSignedNode
+		sigNode, err = node.MultiSignNode([]signature.Signer{nodeSigner}, registry.RegisterNodeSignatureContext, nod)
+		require.NoError(err, "MultiSignNode")
+		err = regState.SetNode(ctx, nil, nod, sigNode)
+		require.NoError(err, "SetNode")
+
+		testNodes = append(testNodes, nod)
+	}
+
+	// Signers have no stake.
+	err = onRuntimeIncorrectResults(
+		ctx,
+		[]signature.PublicKey{testNodes[0].ID},
+		[]signature.PublicKey{testNodes[1].ID},
+		runtime.ID,
+		amount,
+	)
+	require.NoError(err, "should not fail when entity to be slashed has no stake")
+
+	// Add stake.
+	initialEscrow := quantity.NewFromUint64(200)
+	for _, nod := range testNodes {
+		addr := staking.NewAddress(nod.EntityID)
+		var totalShares quantity.Quantity
+		_ = totalShares.FromUint64(200)
+		err = stakeState.SetAccount(ctx, addr, &staking.Account{
+			Escrow: staking.EscrowAccount{
+				Active: staking.SharePool{
+					Balance:     *initialEscrow,
+					TotalShares: totalShares,
+				},
+			},
+		})
+		require.NoError(err, "SetAccount")
+	}
+
+	// TODO: No reward nodes.
+
+	// Multiple slash and reward nodes.
+	var toSlash, toReward []signature.PublicKey
+	for i, nod := range testNodes {
+		if i < numSlashed {
+			toSlash = append(toSlash, nod.ID)
+		} else {
+			toReward = append(toReward, nod.ID)
+		}
+	}
+	err = onRuntimeIncorrectResults(
+		ctx,
+		toSlash,
+		toReward,
+		runtime.ID,
+		amount,
+	)
+	require.NoError(err, "should not fail")
+	runtimePercentage := quantity.NewFromUint64(50)
+	// Ensure nodes were slash, and rewards were distributed.
+	for i, nod := range testNodes {
+		acc, err := stakeState.Account(ctx, staking.NewAddress(nod.EntityID))
+		require.NoError(err, "stakeState.Account")
+
+		expectedEscrow := initialEscrow.Clone()
+		switch i < numSlashed {
+		case true:
+			// Should be slashed.
+			require.NoError(expectedEscrow.Sub(amount), "expectedEscrow.Sub(slashAmount)")
+			require.EqualValues(*expectedEscrow, acc.Escrow.Active.Balance, "Expected amount should be slashed")
+		case false:
+			// Expected reward: ((slashAmount * n_slashed_nodes) / runtimeRewardPercentage) / n_reward_nodes.
+			expectedReward := amount.Clone()
+			require.NoError(expectedReward.Mul(quantity.NewFromUint64(numSlashed)))
+			require.NoError(expectedReward.Mul(runtimePercentage))
+			require.NoError(expectedReward.Quo(quantity.NewFromUint64(100)))
+			require.NoError(expectedReward.Quo(quantity.NewFromUint64(numNodes - numSlashed)))
+			// TODO: Should be escrowed?
+			require.EqualValues(*expectedReward, acc.General.Balance, "Expected amount should be rewarded")
+		}
+	}
+	// Ensure runtime acc got the reward.
+}

go/oasis-node/cmd/registry/runtime/runtime.go

@@ -82,6 +82,7 @@ const (
 
 	// Staking parameters flags.
 	CfgStakingThreshold = "runtime.staking.threshold"
+	CfgStakingSlashing  = "runtime.staking.slashing"
 
 	// List runtimes flags.
 	CfgIncludeSuspended = "include_suspended"
@@ -454,7 +455,27 @@ func runtimeFromFlags() (*registry.Runtime, error) { // nolint: gocyclo
 			rt.Staking.Thresholds[kind] = value
 		}
 	}
+	if sl := viper.GetStringMapString(CfgStakingSlashing); sl != nil {
+		rt.Staking.Slashing = make(map[staking.SlashReason]staking.Slash)
+		for reasonRaw, valueRaw := range sl {
+			var (
+				reason staking.SlashReason
+				value  quantity.Quantity
+			)
 
+			if err = reason.UnmarshalText([]byte(reasonRaw)); err != nil {
+				return nil, fmt.Errorf("staking: bad slash raeson (%s): %w", reasonRaw, err)
+			}
+			if err = value.UnmarshalText([]byte(valueRaw)); err != nil {
+				return nil, fmt.Errorf("staking: bad slash value (%s): %w", valueRaw, err)
+			}
+
+			if _, ok := rt.Staking.Slashing[reason]; ok {
+				return nil, fmt.Errorf("staking: duplicate value for reason '%s'", reason)
+			}
+			rt.Staking.Slashing[reason] = staking.Slash{Amount: value}
+		}
+	}
 	// Validate descriptor.
 	if err = rt.ValidateBasic(true); err != nil {
 		return nil, fmt.Errorf("invalid runtime descriptor: %w", err)
@@ -539,6 +560,7 @@ func init() {
 
 	// Init Staking flags.
 	runtimeFlags.StringToString(CfgStakingThreshold, nil, "Additional staking threshold for this runtime (<kind>=<value>)")
+	runtimeFlags.StringToString(CfgStakingSlashing, nil, "Staking slashing parameter for this runtime (<kind>=<value>)")
 
 	_ = viper.BindPFlags(runtimeFlags)
 	runtimeFlags.AddFlagSet(cmdSigner.Flags)

go/oasis-test-runner/oasis/cli/registry.go

@@ -105,6 +105,14 @@ func (r *RegistryHelpers) runRegistryRuntimeSubcommand(
 			"--"+cmdRegRt.CfgStakingThreshold, fmt.Sprintf("%s=%s", string(kindRaw), string(valueRaw)),
 		)
 	}
+	for reason, value := range runtime.Staking.Slashing {
+		reasonRaw, _ := reason.MarshalText()
+		valueRaw, _ := value.Amount.MarshalText()
+
+		args = append(args,
+			"--"+cmdRegRt.CfgStakingSlashing, fmt.Sprintf("%s=%s", string(reasonRaw), string(valueRaw)),
+		)
+	}
 
 	if out, err := r.runSubCommandWithOutput("registry-runtime-"+cmd, args); err != nil {
 		return fmt.Errorf("failed to run 'registry runtime %s': error: %w output: %s", cmd, err, out.String())