#533 throttle normal operation diff testing PR

tests/difference/core/driver/core_test.go

@@ -22,6 +22,8 @@ import (
 
 	slashingkeeper "github.com/cosmos/cosmos-sdk/x/slashing/keeper"
 	consumerkeeper "github.com/cosmos/interchain-security/x/ccv/consumer/keeper"
+	providerkeeper "github.com/cosmos/interchain-security/x/ccv/provider/keeper"
+	abcitypes "github.com/tendermint/tendermint/abci/types"
 )
 
 type CoreSuite struct {
@@ -72,6 +74,10 @@ func (b *CoreSuite) providerSlashingKeeper() slashingkeeper.Keeper {
 	return b.providerChain().App.(*appProvider.App).SlashingKeeper
 }
 
+func (b *CoreSuite) providerKeeper() providerkeeper.Keeper {
+	return b.providerChain().App.(*appProvider.App).ProviderKeeper
+}
+
 func (b *CoreSuite) consumerKeeper() consumerkeeper.Keeper {
 	return b.consumerChain().App.(*appConsumer.App).ConsumerKeeper
 }
@@ -240,7 +246,9 @@ func (s *CoreSuite) matchState() {
 		// TODO: delegations
 		s.Require().Equalf(int64(s.traces.DelegatorTokens()), s.delegatorBalance(), diagnostic+"P del balance mismatch")
 		for j := 0; j < initState.NumValidators; j++ {
-			s.Require().Equalf(s.traces.Jailed(j) != nil, s.isJailed(int64(j)), diagnostic+"P jail status mismatch for val %d", j)
+			a := s.traces.Jailed(j) != nil
+			b := s.isJailed(int64(j))
+			s.Require().Equalf(a, b, diagnostic+"P jail status mismatch for val %d", j)
 		}
 	}
 	if chain == C {
@@ -258,7 +266,22 @@ func (s *CoreSuite) matchState() {
 	}
 }
 
+// TODO:  debug util, delete before merging to main
+func printValidatorConsAddress(s *CoreSuite, validator int64) {
+	v, found := s.providerStakingKeeper().GetValidator(s.ctx(P), s.validator(validator))
+	if !found {
+		panic("bad")
+	}
+	cons, err := v.GetConsAddr()
+	if err != nil {
+		panic("bad")
+	}
+	vx := abcitypes.Validator{Address: cons.Bytes(), Power: 0}
+	fmt.Println(vx.String())
+}
+
 func (s *CoreSuite) executeTrace() {
+
 	for i := range s.traces.Actions() {
 		s.traces.CurrentActionIx = i
 
@@ -310,6 +333,8 @@ func (s *CoreSuite) TestAssumptions() {
 		s.T().Fatal(FAIL_MSG)
 	}
 
+	// TODO: write assumption that checks that throttle params are appropriate
+
 	// Delegator balance is correct
 	s.Require().Equal(int64(initState.InitialDelegatorTokens), s.delegatorBalance())
 
@@ -428,9 +453,10 @@ func (s *CoreSuite) TestTraces() {
 	s.traces = Traces{
 		Data: LoadTraces("traces.json"),
 	}
-	// s.traces.Data = []TraceData{s.traces.Data[69]}
+	shortest := -1
+	shortestLen := 10000000000
 	for i := range s.traces.Data {
-		s.Run(fmt.Sprintf("Trace num: %d", i), func() {
+		if !s.Run(fmt.Sprintf("Trace num: %d", i), func() {
 			// Setup a new pair of chains for each trace
 			s.SetupTest()
 
@@ -448,13 +474,19 @@ func (s *CoreSuite) TestTraces() {
 			// Record information about the trace, for debugging
 			// diagnostics.
 			s.executeTrace()
-		})
+		}) {
+			if s.traces.CurrentActionIx < shortestLen {
+				shortest = s.traces.CurrentTraceIx
+				shortestLen = s.traces.CurrentActionIx
+			}
+		}
 	}
+	fmt.Println("Shortest [traceIx, actionIx]:", shortest, shortestLen)
+
 }
 
 func TestCoreSuite(t *testing.T) {
-	// TODO: Reenable diff tests once model is updated
-	// suite.Run(t, new(CoreSuite))
+	suite.Run(t, new(CoreSuite))
 }
 
 // SetupTest sets up the test suite in a 'zero' state which matches

tests/difference/core/driver/setup.go

@@ -678,6 +678,13 @@ func (b *Builder) build() {
 
 	b.setSlashParams()
 
+	// TODO: tidy up before merging into main
+	prams := b.providerKeeper().GetParams(b.ctx(P))
+	prams.SlashMeterReplenishFraction = "1.0"
+	prams.SlashMeterReplenishPeriod = time.Second * 1
+	b.providerKeeper().SetParams(b.ctx(P), prams)
+	b.providerKeeper().InitializeSlashMeter(b.ctx(P))
+
 	// Set light client params to match model
 	tmConfig := ibctesting.NewTendermintConfig()
 	tmConfig.UnbondingPeriod = b.initState.UnbondingP

tests/difference/core/model/src/common.ts

@@ -181,6 +181,7 @@ type ModelInitState = {
     downtimeSlashAcks: number[];
     tombstoned: boolean[];
     matureUnbondingOps: number[];
+    queue: (Slash | VscMatured)[];
   };
 };
 

tests/difference/core/model/src/constants.ts

@@ -56,6 +56,7 @@ const MODEL_INIT_STATE: ModelInitState = {
     downtimeSlashAcks: [],
     tombstoned: [false, false, false, false],
     matureUnbondingOps: [],
+    queue: [],
   },
   staking: {
     delegation: [4000, 3000, 2000, 1000],

tests/difference/core/model/src/model.ts

@@ -374,6 +374,8 @@ class CCVProvider {
   tombstoned: boolean[];
   // unbonding operations to be completed in EndBlock
   matureUnbondingOps: number[];
+  // queue
+  queue: (Slash | VscMatured)[];
 
   constructor(model: Model, { ccvP }: ModelInitState) {
     this.m = model;
@@ -382,6 +384,7 @@ class CCVProvider {
 
   endBlockCIS = () => {
     this.vscIDtoH[this.vscID] = this.m.h[P] + 1;
+    this.processPackets();
   };
 
   endBlockVSU = () => {
@@ -420,14 +423,32 @@ class CCVProvider {
   };
 
   onReceive = (data: PacketData) => {
-    // It's sufficient to use isDowntime field as differentiator
-    if ('isDowntime' in data) {
-      this.onReceiveSlash(data);
+    /*
+    TODO: tidy up before merging to main
+    This is some quick prototyping to get the tests passing
+    We have 1 consumer chain so the slash queue is the global queue
+    if the queue is empty we can just process the packet.
+    */
+    if (this.queue.length == 0 && !('isDowntime' in data)) {
+      // Skip the queue
+      this.onReceiveVSCMatured(data as VscMatured);
     } else {
-      this.onReceiveVSCMatured(data);
+      this.queue.push(data);
     }
   };
 
+  processPackets = () => {
+    this.queue.forEach((data) => {
+      // It's sufficient to use isDowntime field as differentiator
+      if ('isDowntime' in data) {
+        this.onReceiveSlash(data);
+      } else {
+        this.onReceiveVSCMatured(data);
+      }
+    });
+    this.queue = [];
+  };
+
   onReceiveVSCMatured = (data: VscMatured) => {
     if (this.vscIDtoOpIDs.has(data.vscID)) {
       this.vscIDtoOpIDs.get(data.vscID)!.forEach((opID: number) => {

tests/e2e/throttle.go

@@ -5,8 +5,10 @@ import (
 
 	sdktypes "github.com/cosmos/cosmos-sdk/types"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
+	channeltypes "github.com/cosmos/ibc-go/v3/modules/core/04-channel/types"
 	icstestingutils "github.com/cosmos/interchain-security/testutil/ibc_testing"
 	providertypes "github.com/cosmos/interchain-security/x/ccv/provider/types"
+	ccvtypes "github.com/cosmos/interchain-security/x/ccv/types"
 	tmtypes "github.com/tendermint/tendermint/types"
 )
 
@@ -203,7 +205,7 @@ func (s *CCVTestSuite) TestMultiConsumerSlashPacketThrottling() {
 
 	// Confirm that the slash packet and trailing VSC matured packet
 	// were handled immediately for the first consumer (this packet was recv first).
-	s.confirmValidatorJailed(valsToSlash[0])
+	s.confirmValidatorJailed(valsToSlash[0], true)
 	s.Require().Equal(uint64(0), providerKeeper.GetPendingPacketDataSize(
 		s.providerCtx(), senderBundles[0].Chain.ChainID))
 
@@ -246,7 +248,7 @@ func (s *CCVTestSuite) TestMultiConsumerSlashPacketThrottling() {
 	// Now all 3 expected vals are jailed, and there are no more queued
 	// slash/vsc matured packets.
 	for _, val := range valsToSlash {
-		s.confirmValidatorJailed(val)
+		s.confirmValidatorJailed(val, true)
 	}
 	s.Require().Equal(uint64(0), providerKeeper.GetPendingPacketDataSize(
 		s.providerCtx(), senderBundles[0].Chain.ChainID))
@@ -339,14 +341,149 @@ func (s *CCVTestSuite) TestSlashMeterAllowanceChanges() {
 
 }
 
-func (s *CCVTestSuite) confirmValidatorJailed(tmVal tmtypes.Validator) {
+// TestSlashSameValidator tests the edge case that that the total slashed validator power
+// queued up for a single block exceeds the slash meter allowance,
+// but some of the slash packets are for the same validator, and therefore some packets
+// will be applied to a validator that is already jailed but still not unbonded (ie. still slashable).
+func (s *CCVTestSuite) TestSlashSameValidator() {
+
+	s.SetupAllCCVChannels()
+
+	// Setup 4 validators with 25% of the total power each.
+	s.setupValidatorPowers()
+
+	providerKeeper := s.providerApp.GetProviderKeeper()
+
+	// Set replenish fraction to 1.0 so that all sent packets should handled immediately (no throttling)
+	params := providerKeeper.GetParams(s.providerCtx())
+	params.SlashMeterReplenishFraction = "1.0"
+	providerKeeper.SetParams(s.providerCtx(), params)
+	providerKeeper.InitializeSlashMeter(s.providerCtx())
+
+	// Send a downtime and double-sign slash packet for 3/4 validators
+	// This will have a total slashing power of 150% total power.
+	tmval1 := s.providerChain.Vals.Validators[1]
+	tmval2 := s.providerChain.Vals.Validators[2]
+	tmval3 := s.providerChain.Vals.Validators[3]
+	s.setDefaultValSigningInfo(*tmval1)
+	s.setDefaultValSigningInfo(*tmval2)
+	s.setDefaultValSigningInfo(*tmval3)
+
+	packets := []channeltypes.Packet{
+		s.constructSlashPacketFromConsumer(s.getFirstBundle(), *tmval1, stakingtypes.Downtime, 1),
+		s.constructSlashPacketFromConsumer(s.getFirstBundle(), *tmval2, stakingtypes.Downtime, 2),
+		s.constructSlashPacketFromConsumer(s.getFirstBundle(), *tmval3, stakingtypes.Downtime, 3),
+		s.constructSlashPacketFromConsumer(s.getFirstBundle(), *tmval1, stakingtypes.DoubleSign, 4),
+		s.constructSlashPacketFromConsumer(s.getFirstBundle(), *tmval2, stakingtypes.DoubleSign, 5),
+		s.constructSlashPacketFromConsumer(s.getFirstBundle(), *tmval3, stakingtypes.DoubleSign, 6),
+	}
+
+	// Recv and queue all slash packets.
+	for _, packet := range packets {
+		slashPacketData := ccvtypes.SlashPacketData{}
+		ccvtypes.ModuleCdc.MustUnmarshalJSON(packet.GetData(), &slashPacketData)
+		providerKeeper.OnRecvSlashPacket(s.providerCtx(), packet, slashPacketData)
+	}
+
+	// We should have 6 pending slash packet entries queued.
+	s.Require().Len(providerKeeper.GetAllPendingSlashPacketEntries(s.providerCtx()), 6)
+
+	// Call next block to process all pending slash packets in end blocker.
+	s.providerChain.NextBlock()
+
+	// All slash packets should have been handled immediately, even though they totaled to 150% of total power.
+	s.Require().Len(providerKeeper.GetAllPendingSlashPacketEntries(s.providerCtx()), 0)
+}
+
+// Similar to TestSlashSameValidator, but 100% of val power is jailed a single block,
+// and in the first packets recv for that block.
+// This edge case should not occur in practice, but is useful to validate that
+// the slash meter can allow any number of slash packets to be handled in a single block when
+// its allowance is set to "1.0".
+func (s CCVTestSuite) TestSlashAllValidators() {
+
+	s.SetupAllCCVChannels()
+
+	// Setup 4 validators with 25% of the total power each.
+	s.setupValidatorPowers()
+
+	providerKeeper := s.providerApp.GetProviderKeeper()
+
+	// Set replenish fraction to 1.0 so that all sent packets should be handled immediately (no throttling)
+	params := providerKeeper.GetParams(s.providerCtx())
+	params.SlashMeterReplenishFraction = "1.0"
+	providerKeeper.SetParams(s.providerCtx(), params)
+	providerKeeper.InitializeSlashMeter(s.providerCtx())
+
+	// The packets to be recv in a single block, ordered as they will be recv.
+	packets := []channeltypes.Packet{}
+
+	// Track and increment ibc seq num for each packet, since these need to be unique.
+	ibcSeqNum := uint64(1)
+
+	// Instantiate a slash packet for each validator,
+	// these first 4 packets should jail 100% of the total power.
+	for _, val := range s.providerChain.Vals.Validators {
+		s.setDefaultValSigningInfo(*val)
+		packets = append(packets, s.constructSlashPacketFromConsumer(
+			s.getFirstBundle(), *val, stakingtypes.Downtime, ibcSeqNum))
+		ibcSeqNum++
+	}
+
+	// add 5 more slash packets for each validator, that will be handled in the same block.
+	for idx, val := range s.providerChain.Vals.Validators {
+		// Set infraction type based on even/odd index.
+		var infractionType stakingtypes.InfractionType
+		if idx%2 == 0 {
+			infractionType = stakingtypes.Downtime
+		} else {
+			infractionType = stakingtypes.DoubleSign
+		}
+		for i := 0; i < 5; i++ {
+			packets = append(packets, s.constructSlashPacketFromConsumer(
+				s.getFirstBundle(), *val, infractionType, ibcSeqNum))
+			ibcSeqNum++
+		}
+	}
+
+	// Recv and queue all slash packets.
+	for _, packet := range packets {
+		slashPacketData := ccvtypes.SlashPacketData{}
+		ccvtypes.ModuleCdc.MustUnmarshalJSON(packet.GetData(), &slashPacketData)
+		providerKeeper.OnRecvSlashPacket(s.providerCtx(), packet, slashPacketData)
+	}
+
+	// We should have 24 pending slash packet entries queued.
+	s.Require().Len(providerKeeper.GetAllPendingSlashPacketEntries(s.providerCtx()), 24)
+
+	// Call next block to process all pending slash packets in end blocker.
+	s.providerChain.NextBlock()
+
+	// All slash packets should have been handled immediately,
+	// even though the first 4 packets jailed 100% of the total power.
+	s.Require().Len(providerKeeper.GetAllPendingSlashPacketEntries(s.providerCtx()), 0)
+
+	// Sanity check that all validators are jailed.
+	for _, val := range s.providerChain.Vals.Validators {
+		// Do not check power, since val power is not yet updated by staking endblocker.
+		s.confirmValidatorJailed(*val, false)
+	}
+
+	// Nextblock would fail the test now, since ibctesting fails when
+	// "applying the validator changes would result in empty set".
+}
+
+func (s *CCVTestSuite) confirmValidatorJailed(tmVal tmtypes.Validator, checkPower bool) {
 	sdkVal, found := s.providerApp.GetE2eStakingKeeper().GetValidator(
 		s.providerCtx(), sdktypes.ValAddress(tmVal.Address))
 	s.Require().True(found)
-	valPower := s.providerApp.GetE2eStakingKeeper().GetLastValidatorPower(
-		s.providerCtx(), sdkVal.GetOperator())
-	s.Require().Equal(int64(0), valPower)
 	s.Require().True(sdkVal.IsJailed())
+
+	if checkPower {
+		valPower := s.providerApp.GetE2eStakingKeeper().GetLastValidatorPower(
+			s.providerCtx(), sdkVal.GetOperator())
+		s.Require().Equal(int64(0), valPower)
+	}
 }
 
 func (s *CCVTestSuite) confirmValidatorNotJailed(tmVal tmtypes.Validator, expectedPower int64) {