Merge pull request #7225 from Agoric/mhofman/5542-export-state-sync

feat: Support state-sync

go.mod

@@ -145,7 +145,7 @@ replace google.golang.org/grpc => google.golang.org/grpc v1.33.2
 replace github.com/tendermint/tendermint => github.com/agoric-labs/tendermint v0.34.23-alpha.agoric.3
 
 // We need a fork of cosmos-sdk until all of the differences are merged.
-replace github.com/cosmos/cosmos-sdk => github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230320225042-2109765fd835
+replace github.com/cosmos/cosmos-sdk => github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230323035240-8551678e04db
 
 replace github.com/cosmos/gaia/v7 => github.com/Agoric/ag0/v7 v7.0.2-alpha.agoric.1
 

go.sum

@@ -80,8 +80,8 @@ github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:
 github.com/adlio/schema v1.3.3 h1:oBJn8I02PyTB466pZO1UZEn1TV5XLlifBSyMrmHl/1I=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
-github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230320225042-2109765fd835 h1:Mmw52cHAUNwtaNXpk7b3lTeoCRd5Vw9Fdrly5ABIxCA=
-github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230320225042-2109765fd835/go.mod h1:fdXvzy+wmYB+W+N139yb0+szbT7zAGgUjmxm5DBrjto=
+github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230323035240-8551678e04db h1:FwZL+MsLF3sLqawUJI3etRSqrxxPEvqdrWqTEQTO18Y=
+github.com/agoric-labs/cosmos-sdk v0.45.11-alpha.agoric.1.0.20230323035240-8551678e04db/go.mod h1:fdXvzy+wmYB+W+N139yb0+szbT7zAGgUjmxm5DBrjto=
 github.com/agoric-labs/cosmos-sdk/ics23/go v0.8.0-alpha.agoric.1 h1:2jvHI/2d+psWAZy6FQ0vXJCHUtfU3ZbbW+pQFL04arQ=
 github.com/agoric-labs/cosmos-sdk/ics23/go v0.8.0-alpha.agoric.1/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg=
 github.com/agoric-labs/tendermint v0.34.23-alpha.agoric.3 h1:aq6F1r3RQkKUYNeMNjRxgGn3dayvKnDK/R6gQF0WoFs=

golang/cosmos/app/app.go

@@ -109,6 +109,7 @@ import (
 	"github.com/Agoric/agoric-sdk/golang/cosmos/x/lien"
 	"github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset"
 	swingsetclient "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/client"
+	swingsetkeeper "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/keeper"
 	swingsettypes "github.com/Agoric/agoric-sdk/golang/cosmos/x/swingset/types"
 	"github.com/Agoric/agoric-sdk/golang/cosmos/x/vbank"
 	vbanktypes "github.com/Agoric/agoric-sdk/golang/cosmos/x/vbank/types"
@@ -227,11 +228,12 @@ type GaiaApp struct { // nolint: golint
 	FeeGrantKeeper feegrantkeeper.Keeper
 	AuthzKeeper    authzkeeper.Keeper
 
-	SwingSetKeeper swingset.Keeper
-	VstorageKeeper vstorage.Keeper
-	VibcKeeper     vibc.Keeper
-	VbankKeeper    vbank.Keeper
-	LienKeeper     lien.Keeper
+	SwingSetKeeper      swingset.Keeper
+	SwingSetSnapshotter swingset.Snapshotter
+	VstorageKeeper      vstorage.Keeper
+	VibcKeeper          vibc.Keeper
+	VbankKeeper         vbank.Keeper
+	LienKeeper          lien.Keeper
 
 	// make scoped keepers public for test purposes
 	ScopedIBCKeeper      capabilitykeeper.ScopedKeeper
@@ -447,6 +449,12 @@ func NewAgoricApp(
 		callToController,
 	)
 
+	app.SwingSetSnapshotter = swingsetkeeper.NewSwingsetSnapshotter(
+		bApp,
+		app.SwingSetKeeper,
+		sendToController,
+	)
+
 	app.VibcKeeper = vibc.NewKeeper(
 		appCodec, keys[vibc.StoreKey],
 		app.IBCKeeper.ChannelKeeper, &app.IBCKeeper.PortKeeper,
@@ -753,6 +761,12 @@ func NewAgoricApp(
 	app.ScopedVibcKeeper = scopedVibcKeeper
 	app.ScopedTransferKeeper = scopedTransferKeeper
 	app.ScopedICAHostKeeper = scopedICAHostKeeper
+	snapshotManager := app.SnapshotManager()
+	if snapshotManager != nil {
+		if err = snapshotManager.RegisterExtensions(&app.SwingSetSnapshotter); err != nil {
+			panic(fmt.Errorf("failed to register snapshot extension: %s", err))
+		}
+	}
 
 	return app
 }
@@ -847,12 +861,34 @@ func (app *GaiaApp) InitChainer(ctx sdk.Context, req abci.RequestInitChain) abci
 
 // Commit tells the controller that the block is commited
 func (app *GaiaApp) Commit() abci.ResponseCommit {
+	err := app.SwingSetSnapshotter.WaitUntilSnapshotStarted()
+
+	if err != nil {
+		app.Logger().Error("swingset snapshot failed to start", "err", err)
+	}
+
 	// Frontrun the BaseApp's Commit method
-	err := swingset.CommitBlock(app.SwingSetKeeper)
+	err = swingset.CommitBlock(app.SwingSetKeeper)
 	if err != nil {
 		panic(err.Error())
 	}
-	return app.BaseApp.Commit()
+
+	res, snapshotHeight := app.BaseApp.CommitWithoutSnapshot()
+
+	err = swingset.AfterCommitBlock(app.SwingSetKeeper)
+	if err != nil {
+		panic(err.Error())
+	}
+
+	if snapshotHeight > 0 {
+		err = app.SwingSetSnapshotter.InitiateSnapshot(snapshotHeight)
+
+		if err != nil {
+			app.Logger().Error("failed to initiate swingset snapshot", "err", err)
+		}
+	}
+
+	return res
 }
 
 // LoadHeight loads a particular height

golang/cosmos/daemon/cmd/root.go

@@ -232,19 +232,14 @@ func (ac appCreator) newApp(
 		panic(err)
 	}
 
-	// FIXME: Actually use the snapshotStore once we have a way to put SwingSet
-	// state into it.
-	var snapshotStore *snapshots.Store
-	if false {
-		snapshotDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots")
-		snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir)
-		if err != nil {
-			panic(err)
-		}
-		snapshotStore, err = snapshots.NewStore(snapshotDB, snapshotDir)
-		if err != nil {
-			panic(err)
-		}
+	snapshotDir := filepath.Join(cast.ToString(appOpts.Get(flags.FlagHome)), "data", "snapshots")
+	snapshotDB, err := sdk.NewLevelDB("metadata", snapshotDir)
+	if err != nil {
+		panic(err)
+	}
+	snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir)
+	if err != nil {
+		panic(err)
 	}
 
 	return gaia.NewAgoricApp(

golang/cosmos/proto/agoric/swingset/swingset.proto

@@ -149,3 +149,17 @@ message Egress {
         (gogoproto.moretags)   = "yaml:\"powerFlags\""
     ];
 }
+
+// The payload messages used by swingset state-sync 
+message ExtensionSnapshotterArtifactPayload {
+    option (gogoproto.equal) = false;
+    string name = 1 [
+        (gogoproto.jsontag)    = "name",
+        (gogoproto.moretags)   = "yaml:\"name\""
+    ];
+
+    bytes data = 2 [
+        (gogoproto.jsontag)    = "data",
+        (gogoproto.moretags)   = "yaml:\"data\""
+    ];
+}

golang/cosmos/x/swingset/abci.go

@@ -1,8 +1,8 @@
 package swingset
 
 import (
-	// "fmt"
 	// "os"
+	"fmt"
 	"time"
 
 	"github.com/cosmos/cosmos-sdk/telemetry"
@@ -101,3 +101,21 @@ func CommitBlock(keeper Keeper) error {
 	}
 	return err
 }
+
+func AfterCommitBlock(keeper Keeper) error {
+	// defer telemetry.ModuleMeasureSince(types.ModuleName, time.Now(), "commit_blocker")
+
+	action := &commitBlockAction{
+		Type:        "AFTER_COMMIT_BLOCK",
+		BlockHeight: endBlockHeight,
+		BlockTime:   endBlockTime,
+	}
+	_, err := keeper.BlockingSend(sdk.Context{}, action)
+
+	// fmt.Fprintf(os.Stderr, "AFTER_COMMIT_BLOCK Returned from SwingSet: %s, %v\n", out, err)
+	if err != nil {
+		// Panic here, in the hopes that a replay from scratch will fix the problem.
+		panic(fmt.Errorf("AFTER_COMMIT_BLOCK failed: %w. Swingset is in an irrecoverable inconsistent state", err))
+	}
+	return err
+}

golang/cosmos/x/swingset/alias.go

@@ -22,6 +22,7 @@ var (
 
 type (
 	Keeper            = keeper.Keeper
+	Snapshotter       = keeper.SwingsetSnapshotter
 	Egress            = types.Egress
 	MsgDeliverInbound = types.MsgDeliverInbound
 	MsgProvision      = types.MsgProvision

golang/cosmos/x/swingset/keeper/keeper.go

@@ -25,13 +25,13 @@ import (
 // Top-level paths for chain storage should remain synchronized with
 // packages/internal/src/chain-storage-paths.js
 const (
-	StoragePathActionQueue  = "actionQueue"
-	StoragePathActivityhash = "activityhash"
-	StoragePathBeansOwing   = "beansOwing"
-	StoragePathEgress       = "egress"
-	StoragePathMailbox      = "mailbox"
-	StoragePathCustom       = "published"
-	StoragePathBundles      = "bundles"
+	StoragePathActionQueue = "actionQueue"
+	StoragePathBeansOwing  = "beansOwing"
+	StoragePathEgress      = "egress"
+	StoragePathMailbox     = "mailbox"
+	StoragePathCustom      = "published"
+	StoragePathBundles     = "bundles"
+	StoragePathSwingStore  = "swingStore"
 )
 
 // 2 ** 256 - 1
@@ -443,6 +443,10 @@ func (k Keeper) SetMailbox(ctx sdk.Context, peer string, mailbox string) {
 	k.vstorageKeeper.LegacySetStorageAndNotify(ctx, vstoragetypes.NewStorageEntry(path, mailbox))
 }
 
+func (k Keeper) ExportSwingStore(ctx sdk.Context) []*vstoragetypes.DataEntry {
+	return k.vstorageKeeper.ExportStorageFromPrefix(ctx, StoragePathSwingStore)
+}
+
 func (k Keeper) PathToEncodedKey(path string) []byte {
 	return k.vstorageKeeper.PathToEncodedKey(path)
 }