Skip to content

Commit

Permalink
fix!: Capability Issue on Restart, Backport to v0.43 (#9836)
Browse files Browse the repository at this point in the history
<!--
The default pull request template is for types feat, fix, or refactor.
For other templates, add one of the following parameters to the url:
- template=docs.md
- template=other.md
-->

## Description

Closes: #9800

The following is a breaking fix to #9800. In the non-breaking fix, the initialization logic was moved out of `InitializeAndSeal` but the API was preserved. I've now removed `InitializeAndSeal` and replaced it with just the `Seal` function, which may be called much more cleanly in `app.go`

---

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [x] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [x] added `!` to the type prefix if API or client breaking change
- [x] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting))
- [x] provided a link to the relevant issue or specification
- [x] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules)
- [x] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing)
- [x] added a changelog entry to `CHANGELOG.md`
- [x] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [x] updated the relevant documentation or specification
- [x] reviewed "Files changed" and left comments if necessary
- [ ] confirmed all CI checks have passed

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed 
- [ ] reviewed state machine logic
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)
  • Loading branch information
AdityaSripal authored Aug 3, 2021
1 parent bc201f8 commit 7f316ad
Show file tree
Hide file tree
Showing 10 changed files with 234 additions and 58 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
* [\#9432](https://github.com/cosmos/cosmos-sdk/pull/9432) `ConsensusParamsKeyTable` moved from `params/keeper` to `params/types`
* [\#9576](https://github.com/cosmos/cosmos-sdk/pull/9576) Add debug error message to `sdkerrors.QueryResult` when enabled
* [\#9650](https://github.com/cosmos/cosmos-sdk/pull/9650) Removed deprecated message handler implementation from the SDK modules.
* (x/capability) [\#9836](https://github.com/cosmos/cosmos-sdk/pull/9836) Removed `InitializeAndSeal(ctx sdk.Context)` and replaced with `Seal()`.

### Client Breaking Changes

Expand Down Expand Up @@ -94,6 +95,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
* (server) [#9704](https://github.com/cosmos/cosmos-sdk/pull/9704) Start GRPCWebServer in goroutine, avoid blocking other services from starting.
* [\#9762](https://github.com/cosmos/cosmos-sdk/pull/9762) The init command uses the chain-id from the client config if --chain-id is not provided
* [\#9793](https://github.com/cosmos/cosmos-sdk/pull/9793) Fixed ECDSA/secp256r1 transaction malleability.
* [\#9836](https://github.com/cosmos/cosmos-sdk/pull/9836) Fixes capability initialization issue on tx revert by moving initialization logic to `InitChain` and `BeginBlock`.

### State Machine Breaking

Expand Down
21 changes: 8 additions & 13 deletions simapp/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
tmos "github.com/tendermint/tendermint/libs/os"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
dbm "github.com/tendermint/tm-db"

"github.com/cosmos/cosmos-sdk/baseapp"
Expand Down Expand Up @@ -208,7 +207,9 @@ func NewSimApp(
authzkeeper.StoreKey,
)
tkeys := sdk.NewTransientStoreKeys(paramstypes.TStoreKey)
memKeys := sdk.NewMemoryStoreKeys(capabilitytypes.MemStoreKey)
// NOTE: The testingkey is just mounted for testing purposes. Actual applications should
// not include this key.
memKeys := sdk.NewMemoryStoreKeys(capabilitytypes.MemStoreKey, "testingkey")

app := &SimApp{
BaseApp: bApp,
Expand All @@ -227,6 +228,9 @@ func NewSimApp(
bApp.SetParamStore(app.ParamsKeeper.Subspace(baseapp.Paramspace).WithKeyTable(paramstypes.ConsensusParamsKeyTable()))

app.CapabilityKeeper = capabilitykeeper.NewKeeper(appCodec, keys[capabilitytypes.StoreKey], memKeys[capabilitytypes.MemStoreKey])
// Applications that wish to enforce statically created ScopedKeepers should call `Seal` after creating
// their scoped modules in `NewApp` with `ScopeToModule`
app.CapabilityKeeper.Seal()

// add keepers
app.AccountKeeper = authkeeper.NewAccountKeeper(
Expand Down Expand Up @@ -322,8 +326,9 @@ func NewSimApp(
// there is nothing left over in the validator fee pool, so as to keep the
// CanWithdrawInvariant invariant.
// NOTE: staking module is required if HistoricalEntries param > 0
// NOTE: capability module's beginblocker must come before any modules using capabilities (e.g. IBC)
app.mm.SetOrderBeginBlockers(
upgradetypes.ModuleName, minttypes.ModuleName, distrtypes.ModuleName, slashingtypes.ModuleName,
upgradetypes.ModuleName, capabilitytypes.ModuleName, minttypes.ModuleName, distrtypes.ModuleName, slashingtypes.ModuleName,
evidencetypes.ModuleName, stakingtypes.ModuleName,
)
app.mm.SetOrderEndBlockers(crisistypes.ModuleName, govtypes.ModuleName, stakingtypes.ModuleName)
Expand Down Expand Up @@ -399,16 +404,6 @@ func NewSimApp(
if err := app.LoadLatestVersion(); err != nil {
tmos.Exit(err.Error())
}

// Initialize and seal the capability keeper so all persistent capabilities
// are loaded in-memory and prevent any further modules from creating scoped
// sub-keepers.
// This must be done during creation of baseapp rather than in InitChain so
// that in-memory capabilities get regenerated on app restart.
// Note that since this reads from the store, we can only perform it when
// `loadLatest` is set to true.
ctx := app.BaseApp.NewUncachedContext(true, tmproto.Header{})
app.CapabilityKeeper.InitializeAndSeal(ctx)
}

return app
Expand Down
21 changes: 21 additions & 0 deletions x/capability/abci.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package capability

import (
"time"

"github.com/cosmos/cosmos-sdk/telemetry"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/x/capability/keeper"
"github.com/cosmos/cosmos-sdk/x/capability/types"
)

// BeginBlocker will call InitMemStore to initialize the memory stores in the case
// that this is the first time the node is executing a block since restarting (wiping memory).
// In this case, the BeginBlocker method will reinitialize the memory stores locally, so that subsequent
// capability transactions will pass.
// Otherwise BeginBlocker performs a no-op.
func BeginBlocker(ctx sdk.Context, k keeper.Keeper) {
defer telemetry.ModuleMeasureSince(types.ModuleName, time.Now(), telemetry.MetricKeyBeginBlocker)

k.InitMemStore(ctx)
}
97 changes: 97 additions & 0 deletions x/capability/capability_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package capability_test

import (
"testing"

"github.com/stretchr/testify/suite"
abci "github.com/tendermint/tendermint/abci/types"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"

"github.com/cosmos/cosmos-sdk/codec"
"github.com/cosmos/cosmos-sdk/simapp"
sdk "github.com/cosmos/cosmos-sdk/types"
"github.com/cosmos/cosmos-sdk/types/module"
banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
"github.com/cosmos/cosmos-sdk/x/capability"
"github.com/cosmos/cosmos-sdk/x/capability/keeper"
"github.com/cosmos/cosmos-sdk/x/capability/types"
)

type CapabilityTestSuite struct {
suite.Suite

cdc codec.Codec
ctx sdk.Context
app *simapp.SimApp
keeper *keeper.Keeper
module module.AppModule
}

func (suite *CapabilityTestSuite) SetupTest() {
checkTx := false
app := simapp.Setup(checkTx)
cdc := app.AppCodec()

// create new keeper so we can define custom scoping before init and seal
keeper := keeper.NewKeeper(cdc, app.GetKey(types.StoreKey), app.GetMemKey(types.MemStoreKey))

suite.app = app
suite.ctx = app.BaseApp.NewContext(checkTx, tmproto.Header{Height: 1})
suite.keeper = keeper
suite.cdc = cdc
suite.module = capability.NewAppModule(cdc, *keeper)
}

// The following test case mocks a specific bug discovered in https://github.com/cosmos/cosmos-sdk/issues/9800
// and ensures that the current code successfully fixes the issue.
func (suite *CapabilityTestSuite) TestInitializeMemStore() {
sk1 := suite.keeper.ScopeToModule(banktypes.ModuleName)

cap1, err := sk1.NewCapability(suite.ctx, "transfer")
suite.Require().NoError(err)
suite.Require().NotNil(cap1)

// mock statesync by creating new keeper that shares persistent state but loses in-memory map
newKeeper := keeper.NewKeeper(suite.cdc, suite.app.GetKey(types.StoreKey), suite.app.GetMemKey("testingkey"))
newSk1 := newKeeper.ScopeToModule(banktypes.ModuleName)

// Mock App startup
ctx := suite.app.BaseApp.NewUncachedContext(false, tmproto.Header{})
newKeeper.Seal()
suite.Require().False(newKeeper.IsInitialized(ctx), "memstore initialized flag set before BeginBlock")

// Mock app beginblock and ensure that no gas has been consumed and memstore is initialized
ctx = suite.app.BaseApp.NewContext(false, tmproto.Header{}).WithBlockGasMeter(sdk.NewGasMeter(50))
prevGas := ctx.BlockGasMeter().GasConsumed()
restartedModule := capability.NewAppModule(suite.cdc, *newKeeper)
restartedModule.BeginBlock(ctx, abci.RequestBeginBlock{})
suite.Require().True(newKeeper.IsInitialized(ctx), "memstore initialized flag not set")
gasUsed := ctx.BlockGasMeter().GasConsumed()

suite.Require().Equal(prevGas, gasUsed, "beginblocker consumed gas during execution")

// Mock the first transaction getting capability and subsequently failing
// by using a cached context and discarding all cached writes.
cacheCtx, _ := ctx.CacheContext()
_, ok := newSk1.GetCapability(cacheCtx, "transfer")
suite.Require().True(ok)

// Ensure that the second transaction can still receive capability even if first tx fails.
ctx = suite.app.BaseApp.NewContext(false, tmproto.Header{})

cap1, ok = newSk1.GetCapability(ctx, "transfer")
suite.Require().True(ok)

// Ensure the capabilities don't get reinitialized on next BeginBlock
// by testing to see if capability returns same pointer
// also check that initialized flag is still set
restartedModule.BeginBlock(ctx, abci.RequestBeginBlock{})
recap, ok := newSk1.GetCapability(ctx, "transfer")
suite.Require().True(ok)
suite.Require().Equal(cap1, recap, "capabilities got reinitialized after second BeginBlock")
suite.Require().True(newKeeper.IsInitialized(ctx), "memstore initialized flag not set")
}

func TestCapabilityTestSuite(t *testing.T) {
suite.Run(t, new(CapabilityTestSuite))
}
5 changes: 3 additions & 2 deletions x/capability/genesis.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@ func InitGenesis(ctx sdk.Context, k keeper.Keeper, genState types.GenesisState)
panic(err)
}

// set owners for each index and initialize capability
// set owners for each index
for _, genOwner := range genState.Owners {
k.SetOwners(ctx, genOwner.Index, genOwner.IndexOwners)
k.InitializeCapability(ctx, genOwner.Index, genOwner.IndexOwners)
}
// initialize in-memory capabilities
k.InitMemStore(ctx)
}

// ExportGenesis returns the capability module's exported genesis.
Expand Down
59 changes: 59 additions & 0 deletions x/capability/genesis_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package capability_test

import (
"github.com/tendermint/tendermint/libs/log"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
dbm "github.com/tendermint/tm-db"

"github.com/cosmos/cosmos-sdk/simapp"
sdk "github.com/cosmos/cosmos-sdk/types"
banktypes "github.com/cosmos/cosmos-sdk/x/bank/types"
"github.com/cosmos/cosmos-sdk/x/capability"
"github.com/cosmos/cosmos-sdk/x/capability/keeper"
"github.com/cosmos/cosmos-sdk/x/capability/types"
stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
)

func (suite *CapabilityTestSuite) TestGenesis() {
sk1 := suite.keeper.ScopeToModule(banktypes.ModuleName)
sk2 := suite.keeper.ScopeToModule(stakingtypes.ModuleName)

cap1, err := sk1.NewCapability(suite.ctx, "transfer")
suite.Require().NoError(err)
suite.Require().NotNil(cap1)

err = sk2.ClaimCapability(suite.ctx, cap1, "transfer")
suite.Require().NoError(err)

cap2, err := sk2.NewCapability(suite.ctx, "ica")
suite.Require().NoError(err)
suite.Require().NotNil(cap2)

genState := capability.ExportGenesis(suite.ctx, *suite.keeper)

// create new app that does not share persistent or in-memory state
// and initialize app from exported genesis state above.
db := dbm.NewMemDB()
encCdc := simapp.MakeTestEncodingConfig()
newApp := simapp.NewSimApp(log.NewNopLogger(), db, nil, true, map[int64]bool{}, simapp.DefaultNodeHome, 5, encCdc, simapp.EmptyAppOptions{})

newKeeper := keeper.NewKeeper(suite.cdc, newApp.GetKey(types.StoreKey), newApp.GetMemKey(types.MemStoreKey))
newSk1 := newKeeper.ScopeToModule(banktypes.ModuleName)
newSk2 := newKeeper.ScopeToModule(stakingtypes.ModuleName)
deliverCtx, _ := newApp.BaseApp.NewUncachedContext(false, tmproto.Header{}).WithBlockGasMeter(sdk.NewInfiniteGasMeter()).CacheContext()

capability.InitGenesis(deliverCtx, *newKeeper, *genState)

// check that all previous capabilities exist in new app after InitGenesis
sk1Cap1, ok := newSk1.GetCapability(deliverCtx, "transfer")
suite.Require().True(ok, "could not get first capability after genesis on first ScopedKeeper")
suite.Require().Equal(*cap1, *sk1Cap1, "capability values not equal on first ScopedKeeper")

sk2Cap1, ok := newSk2.GetCapability(deliverCtx, "transfer")
suite.Require().True(ok, "could not get first capability after genesis on first ScopedKeeper")
suite.Require().Equal(*cap1, *sk2Cap1, "capability values not equal on first ScopedKeeper")

sk2Cap2, ok := newSk2.GetCapability(deliverCtx, "ica")
suite.Require().True(ok, "could not get second capability after genesis on second ScopedKeeper")
suite.Require().Equal(*cap2, *sk2Cap2, "capability values not equal on second ScopedKeeper")
}
74 changes: 35 additions & 39 deletions x/capability/keeper/keeper.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,6 @@ import (
"github.com/cosmos/cosmos-sdk/x/capability/types"
)

// initialized is a global variable used by GetCapability to ensure that the memory store
// and capability map are correctly populated. A state-synced node may copy over all the persistent
// state and start running the application without having the in-memory state required for x/capability.
// Thus, we must initialized the memory stores on-the-fly during tx execution once the first GetCapability
// is called.
// This is a temporary fix and should be replaced by a more robust solution in the next breaking release.
var initialized = false

type (
// Keeper defines the capability module's keeper. It is responsible for provisioning,
// tracking, and authenticating capabilities at runtime. During application
Expand Down Expand Up @@ -97,38 +89,58 @@ func (k *Keeper) ScopeToModule(moduleName string) ScopedKeeper {
}
}

// InitializeAndSeal loads all capabilities from the persistent KVStore into the
// in-memory store and seals the keeper to prevent further modules from creating
// a scoped keeper. InitializeAndSeal must be called once after the application
// state is loaded.
func (k *Keeper) InitializeAndSeal(ctx sdk.Context) {
// Seal seals the keeper to prevent further modules from creating a scoped keeper.
// Seal may be called during app initialization for applications that do not wish to create scoped keepers dynamically.
func (k *Keeper) Seal() {
if k.sealed {
panic("cannot initialize and seal an already sealed capability keeper")
}

k.sealed = true
}

// InitMemStore will initialize the memory store if the initialized flag is not set.
// InitMemStore logic should be run in first `BeginBlock` or `InitChain` (whichever is run on app start)
// so that the memory store is properly initialized before any transactions are run.
// InitMemStore also asserts the memory store type is correct, and will panic if it does not have store type of `StoreTypeMemory`.
func (k *Keeper) InitMemStore(ctx sdk.Context) {
memStore := ctx.KVStore(k.memKey)
memStoreType := memStore.GetStoreType()

if memStoreType != sdk.StoreTypeMemory {
panic(fmt.Sprintf("invalid memory store type; got %s, expected: %s", memStoreType, sdk.StoreTypeMemory))
}

prefixStore := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixIndexCapability)
iterator := sdk.KVStorePrefixIterator(prefixStore, nil)
// create context with no block gas meter to ensure we do not consume gas during local initialization logic.
noGasCtx := ctx.WithBlockGasMeter(sdk.NewInfiniteGasMeter())

// check if memory store has not been initialized yet by checking if initialized flag is nil.
if !k.IsInitialized(noGasCtx) {
prefixStore := prefix.NewStore(noGasCtx.KVStore(k.storeKey), types.KeyPrefixIndexCapability)
iterator := sdk.KVStorePrefixIterator(prefixStore, nil)

// initialize the in-memory store for all persisted capabilities
defer iterator.Close()

// initialize the in-memory store for all persisted capabilities
defer iterator.Close()
for ; iterator.Valid(); iterator.Next() {
index := types.IndexFromKey(iterator.Key())

for ; iterator.Valid(); iterator.Next() {
index := types.IndexFromKey(iterator.Key())
var capOwners types.CapabilityOwners

var capOwners types.CapabilityOwners
k.cdc.MustUnmarshal(iterator.Value(), &capOwners)
k.InitializeCapability(noGasCtx, index, capOwners)
}

k.cdc.MustUnmarshal(iterator.Value(), &capOwners)
k.InitializeCapability(ctx, index, capOwners)
// set the initialized flag so we don't rerun initialization logic
memStore := noGasCtx.KVStore(k.memKey)
memStore.Set(types.KeyMemInitialized, []byte{1})
}
}

k.sealed = true
// IsInitialized returns true if the initialized flag is set, and false otherwise
func (k *Keeper) IsInitialized(ctx sdk.Context) bool {
memStore := ctx.KVStore(k.memKey)
return memStore.Get(types.KeyMemInitialized) != nil
}

// InitializeIndex sets the index to one (or greater) in InitChain according
Expand Down Expand Up @@ -350,22 +362,6 @@ func (sk ScopedKeeper) ReleaseCapability(ctx sdk.Context, cap *types.Capability)
// by name. The module is not allowed to retrieve capabilities which it does not
// own.
func (sk ScopedKeeper) GetCapability(ctx sdk.Context, name string) (*types.Capability, bool) {
// Create a keeper that will set all in-memory mappings correctly into memstore and capmap if scoped keeper is not initialized yet.
// This ensures that the in-memory mappings are correctly filled in, in case this is a state-synced node.
// This is a temporary non-breaking fix, a future PR should store the reverse mapping in the persistent store and reconstruct forward mapping and capmap on the fly.
if !initialized {
// create context with infinite gas meter to avoid app state mismatch.
initCtx := ctx.WithGasMeter(sdk.NewInfiniteGasMeter())
k := Keeper{
cdc: sk.cdc,
storeKey: sk.storeKey,
memKey: sk.memKey,
capMap: sk.capMap,
}
k.InitializeAndSeal(initCtx)
initialized = true
}

if strings.TrimSpace(name) == "" {
return nil, false
}
Expand Down
Loading

0 comments on commit 7f316ad

Please sign in to comment.