Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Save sim params and export app state to file #4731

Merged
merged 16 commits into from
Jul 19, 2019
Merged

Save sim params and export app state to file #4731

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
d264090
save sim params and export app state to file
fedekunze Jul 17, 2019
ba3fddb
leave exportStateHeight for stop/resume PR
fedekunze Jul 18, 2019
9ea5da5
clog
fedekunze Jul 18, 2019
da67ed8
reorder
fedekunze Jul 18, 2019
07cbac5
comments
fedekunze Jul 18, 2019
9395705
merge conflicts
fedekunze Jul 18, 2019
03746b0
address @alexanderbez comments
fedekunze Jul 19, 2019
f8e1a81
Update simapp/sim_test.go
fedekunze Jul 19, 2019
8cad55e
update
fedekunze Jul 19, 2019
342669c
Merge branch 'fedekunze/4566-save-sim-params' of https://github.com/c…
fedekunze Jul 19, 2019
31af201
update makefile
fedekunze Jul 19, 2019
b410367
Apply suggestions from code review
fedekunze Jul 19, 2019
5ce0650
minor changes
fedekunze Jul 19, 2019
4f9c488
Merge branch 'fedekunze/4566-save-sim-params' of https://github.com/c…
fedekunze Jul 19, 2019
3825902
fix param export
fedekunze Jul 19, 2019
b44f967
update doc.go
fedekunze Jul 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .pending/improvements/sdk/4566-sim-export
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#4566 Export simulation's parameters and app state to JSON in order to reproduce bugs
and invariants.
206 changes: 165 additions & 41 deletions simapp/sim_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,28 +31,35 @@ import (
)

func init() {
flag.StringVar(&genesisFile, "SimulationGenesis", "", "custom simulation genesis file; cannot be used with params file")
flag.StringVar(&paramsFile, "SimulationParams", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
flag.Int64Var(&seed, "SimulationSeed", 42, "simulation random seed")
flag.IntVar(&numBlocks, "SimulationNumBlocks", 500, "number of blocks")
flag.IntVar(&blockSize, "SimulationBlockSize", 200, "operations per block")
flag.BoolVar(&enabled, "SimulationEnabled", false, "enable the simulation")
flag.BoolVar(&verbose, "SimulationVerbose", false, "verbose log output")
flag.BoolVar(&lean, "SimulationLean", false, "lean simulation log output")
flag.BoolVar(&commit, "SimulationCommit", false, "have the simulation commit")
flag.IntVar(&period, "SimulationPeriod", 1, "run slow invariants only once every period assertions")
flag.StringVar(&genesisFile, "Genesis", "", "custom simulation genesis file; cannot be used with params file")
flag.StringVar(&paramsFile, "Params", "", "custom simulation params file which overrides any random params; cannot be used with genesis")
flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON")
flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params")
flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON")
flag.Int64Var(&seed, "Seed", 42, "simulation random seed")
flag.IntVar(&numBlocks, "NumBlocks", 500, "number of blocks")
flag.IntVar(&blockSize, "BlockSize", 200, "operations per block")
flag.BoolVar(&enabled, "Enabled", false, "enable the simulation")
flag.BoolVar(&verbose, "Verbose", false, "verbose log output")
flag.BoolVar(&lean, "Lean", false, "lean simulation log output")
flag.BoolVar(&commit, "Commit", false, "have the simulation commit")
flag.IntVar(&period, "Period", 1, "run slow invariants only once every period assertions")
flag.BoolVar(&onOperation, "SimulateEveryOperation", false, "run slow invariants every operation")
flag.BoolVar(&allInvariants, "PrintAllInvariants", false, "print all invariants if a broken invariant is found")
}

// helper function for populating input for SimulateFromSeed
func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sighh...this function is becoming overwhelmingly verbose and hard to grok.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agree... I'd like to do some refactors later on the simulator and this is one of the things that'd be cleaned up

testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
simulation.WeightedOperations, sdk.Invariants, int, int, bool, bool, bool, bool, map[string]bool) {
simulation.WeightedOperations, sdk.Invariants, int, int, int,
bool, bool, bool, bool, bool, map[string]bool) {

exportParams := exportParamsPath != ""

return tb, w, app.BaseApp, appStateFn, seed,
testAndRunTxs(app), invariants(app), numBlocks, blockSize, commit,
lean, onOperation, allInvariants, app.ModuleAccountAddrs()
testAndRunTxs(app), invariants(app),
numBlocks, exportParamsHeight, blockSize,
exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs()
}

func appStateFn(
Expand Down Expand Up @@ -136,6 +143,7 @@ func appStateRandomizedFn(
return appState, accs, "simulation"
}

// TODO: add description
func testAndRunTxs(app *SimApp) []simulation.WeightedOperation {
cdc := MakeCodec()
ap := make(simulation.AppParams)
Expand Down Expand Up @@ -358,12 +366,44 @@ func BenchmarkFullAppSimulation(b *testing.B) {
app := NewSimApp(logger, db, nil, true, 0)

// Run randomized simulation
// TODO parameterize numbers, save for a later PR
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))
if err != nil {
fmt.Println(err)
b.Fail()
// TODO: parameterize numbers, save for a later PR
fedekunze marked this conversation as resolved.
Show resolved Hide resolved
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(b, os.Stdout, app))

// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
if err != nil {
fmt.Println(err)
b.Fail()
}
err = ioutil.WriteFile(exportParamsPath, []byte(appState), 0644)
fedekunze marked this conversation as resolved.
Show resolved Hide resolved
fedekunze marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
fmt.Println(err)
b.Fail()
}
}

if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := app.cdc.MarshalJSONIndent(params, "", " ")
if err != nil {
fmt.Println(err)
b.Fail()
}

err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
if err != nil {
fmt.Println(err)
b.Fail()
}
}

if simErr != nil {
fmt.Println(simErr)
b.FailNow()
}

if commit {
fmt.Println("GoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
Expand Down Expand Up @@ -397,16 +437,36 @@ func TestFullAppSimulation(t *testing.T) {
require.Equal(t, "SimApp", app.Name())

// Run randomized simulation
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))

// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)

err = ioutil.WriteFile(exportParamsPath, []byte(appState), 0644)
fedekunze marked this conversation as resolved.
Show resolved Hide resolved
require.NoError(t, err)
}

if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := app.cdc.MarshalJSONIndent(params, "", " ")
require.NoError(t, err)

err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
require.NoError(t, err)
}

require.NoError(t, simErr)

if commit {
// for memdb:
// fmt.Println("Database Size", db.Stats()["database.size"])
fmt.Println("GoLevelDB Stats")
fmt.Println(db.Stats()["leveldb.stats"])
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}

require.Nil(t, err)
}

func TestAppImportExport(t *testing.T) {
Expand Down Expand Up @@ -434,7 +494,28 @@ func TestAppImportExport(t *testing.T) {
require.Equal(t, "SimApp", app.Name())

// Run randomized simulation
_, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
_, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))

// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)

err = ioutil.WriteFile(exportParamsPath, []byte(appState), 0644)
fedekunze marked this conversation as resolved.
Show resolved Hide resolved
require.NoError(t, err)
}

if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := app.cdc.MarshalJSONIndent(params, "", " ")
require.NoError(t, err)

err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
require.NoError(t, err)
}

require.NoError(t, simErr)

if commit {
// for memdb:
Expand All @@ -444,7 +525,6 @@ func TestAppImportExport(t *testing.T) {
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}

require.Nil(t, err)
fmt.Printf("Exporting genesis...\n")

appState, _, err := app.ExportAppStateAndValidators(false, []string{})
Expand Down Expand Up @@ -530,7 +610,28 @@ func TestAppSimulationAfterImport(t *testing.T) {
require.Equal(t, "SimApp", app.Name())

// Run randomized simulation
stopEarly, err := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))
stopEarly, params, simErr := simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, app))

// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
require.NoError(t, err)

err = ioutil.WriteFile(exportParamsPath, []byte(appState), 0644)
fedekunze marked this conversation as resolved.
Show resolved Hide resolved
require.NoError(t, err)
}

if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := app.cdc.MarshalJSONIndent(params, "", " ")
require.NoError(t, err)

err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
require.NoError(t, err)
}

require.NoError(t, simErr)

if commit {
// for memdb:
Expand All @@ -540,8 +641,6 @@ func TestAppSimulationAfterImport(t *testing.T) {
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
}

require.Nil(t, err)

if stopEarly {
// we can't export or import a zero-validator genesis
fmt.Printf("We can't export or import a zero-validator genesis, exiting test...\n")
Expand Down Expand Up @@ -572,7 +671,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
})

// Run randomized simulation on imported app
_, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
_, _, err = simulation.SimulateFromSeed(getSimulateFromSeedInput(t, os.Stdout, newApp))
require.Nil(t, err)
}

Expand All @@ -597,15 +696,9 @@ func TestAppStateDeterminism(t *testing.T) {
// Run randomized simulation
simulation.SimulateFromSeed(
t, os.Stdout, app.BaseApp, appStateFn, seed,
testAndRunTxs(app),
[]sdk.Invariant{},
50,
100,
true,
false,
false,
false,
app.ModuleAccountAddrs(),
testAndRunTxs(app), []sdk.Invariant{},
50, 100, 0,
false, true, false, false, false, app.ModuleAccountAddrs(),
)
appHash := app.LastCommitID().Hash
appHashList[j] = appHash
Expand All @@ -629,13 +722,44 @@ func BenchmarkInvariants(b *testing.B) {
app := NewSimApp(logger, db, nil, true, 0)

// 2. Run parameterized simulation (w/o invariants)
_, err := simulation.SimulateFromSeed(
_, params, simErr := simulation.SimulateFromSeed(
b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
[]sdk.Invariant{}, numBlocks, blockSize, commit, lean, onOperation, false,
app.ModuleAccountAddrs(),
[]sdk.Invariant{}, numBlocks, exportParamsHeight, blockSize,
exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(),
)
if err != nil {
fmt.Println(err)

// export state and params before the simulation error is checked
if exportStatePath != "" {
fmt.Println("Exporting app state...")
appState, _, err := app.ExportAppStateAndValidators(false, nil)
if err != nil {
fmt.Println(err)
b.Fail()
}
err = ioutil.WriteFile(exportParamsPath, []byte(appState), 0644)
fedekunze marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
fmt.Println(err)
b.Fail()
}
}

if exportParamsPath != "" {
fmt.Println("Exporting simulation params...")
paramsBz, err := app.cdc.MarshalJSONIndent(params, "", " ")
if err != nil {
fmt.Println(err)
b.Fail()
}

err = ioutil.WriteFile(exportParamsPath, paramsBz, 0644)
if err != nil {
fmt.Println(err)
b.Fail()
}
}

if simErr != nil {
fmt.Println(simErr)
b.FailNow()
}

Expand Down
27 changes: 15 additions & 12 deletions simapp/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,21 @@ import (
)

var (
genesisFile string
paramsFile string
seed int64
numBlocks int
blockSize int
enabled bool
verbose bool
lean bool
commit bool
period int
onOperation bool // TODO Remove in favor of binary search for invariant violation
allInvariants bool
genesisFile string
paramsFile string
exportParamsPath string
exportParamsHeight int
exportStatePath string
seed int64
numBlocks int
blockSize int
enabled bool
verbose bool
lean bool
commit bool
period int
onOperation bool // TODO Remove in favor of binary search for invariant violation
allInvariants bool
)

// NewSimAppUNSAFE is used for debugging purposes only.
Expand Down
Loading