diff --git a/go/oasis-node/cmd/common/consensus/consensus.go b/go/oasis-node/cmd/common/consensus/consensus.go index 46708f13e26..86de7cdfc14 100644 --- a/go/oasis-node/cmd/common/consensus/consensus.go +++ b/go/oasis-node/cmd/common/consensus/consensus.go @@ -11,6 +11,7 @@ import ( "github.com/oasislabs/oasis-core/go/common/logging" "github.com/oasislabs/oasis-core/go/consensus/api/transaction" + genesisAPI "github.com/oasislabs/oasis-core/go/genesis/api" genesisFile "github.com/oasislabs/oasis-core/go/genesis/file" cmdCommon "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common" cmdFlags "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/flags" @@ -47,7 +48,7 @@ func AssertTxFileOK() { // XXX: Other checks to see if we can write to the file? } -func InitGenesis() { +func InitGenesis() *genesisAPI.Document { genesis, err := genesisFile.DefaultFileProvider() if err != nil { logger.Error("failed to load genesis file", @@ -66,6 +67,8 @@ func InitGenesis() { os.Exit(1) } genesisDoc.SetChainContext() + + return genesisDoc } func GetTxNonceAndFee() (uint64, *transaction.Fee) { diff --git a/go/oasis-node/cmd/debug/storage/export.go b/go/oasis-node/cmd/debug/storage/export.go new file mode 100644 index 00000000000..42a68928572 --- /dev/null +++ b/go/oasis-node/cmd/debug/storage/export.go @@ -0,0 +1,202 @@ +package storage + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/oasislabs/oasis-core/go/common" + cmdCommon "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common" + cmdConsensus "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/consensus" + "github.com/oasislabs/oasis-core/go/storage" + storageAPI "github.com/oasislabs/oasis-core/go/storage/api" + storageClient "github.com/oasislabs/oasis-core/go/storage/client" + storageDatabase "github.com/oasislabs/oasis-core/go/storage/database" +) + +const cfgExportDir = "storage.export.dir" + +var ( + storageExportCmd = &cobra.Command{ + Use: "export", + Short: "export the storage roots contained in a state dump", + Run: doExport, + } + + storageExportFlags = flag.NewFlagSet("", flag.ContinueOnError) +) + +func doExport(cmd *cobra.Command, args []string) { + var ok bool + defer func() { + if !ok { + os.Exit(1) + } + }() + + if err := cmdCommon.Init(); err != nil { + cmdCommon.EarlyLogAndExit(err) + } + + dataDir := cmdCommon.DataDir() + if dataDir == "" { + logger.Error("data directory must be set") + return + } + + destDir := viper.GetString(cfgExportDir) + if destDir == "" { + destDir = dataDir + } else if err := common.Mkdir(destDir); err != nil { + logger.Error("failed to create destination directory", + "err", err, + "dir", destDir, + ) + return + } + + // Load the genesis document. + genesisDoc := cmdConsensus.InitGenesis() + + // Initialize the storage backend. + storageBackend, err := newDirectStorageBackend(dataDir) + if err != nil { + logger.Error("failed to construct storage backend", + "err", err, + ) + return + } + + logger.Info("waiting for storage backend initialization") + <-storageBackend.Initialized() + defer storageBackend.Cleanup() + + // For each storage root. + for runtimeID, blk := range genesisDoc.RootHash.Blocks { + logger.Info("fetching checkpoint write log", + "runtime_id", runtimeID, + ) + + // Get the checkpoint iterator. + root := storageAPI.Root{ + Namespace: blk.Header.Namespace, + Round: blk.Header.Round, + Hash: blk.Header.StateRoot, + } + it, err := storageBackend.GetCheckpoint(context.Background(), + &storageAPI.GetCheckpointRequest{ + Root: root, + }, + ) + if err != nil { + logger.Error("failed getting checkpoint", + "err", err, + "namespace", root.Namespace, + "round", root.Round, + "root", root.Hash, + ) + return + } + + fn := fmt.Sprintf("storage-dump-%v-%d.json", + runtimeID.String(), + blk.Header.Round, + ) + fn = filepath.Join(destDir, fn) + if err = exportIterator(fn, &root, it); err != nil { + return + } + } + + ok = true +} + +func exportIterator(fn string, root *storageAPI.Root, it storageAPI.WriteLogIterator) error { + // Create the dump file, and initialize a JSON stream encoder. + f, err := os.Create(fn) + if err != nil { + logger.Error("failed to create dump file", + "err", err, + "fn", fn, + ) + return err + } + defer f.Close() + + w := bufio.NewWriter(f) + defer w.Flush() + + enc := json.NewEncoder(w) + + // Dump the root. + if err = enc.Encode(root); err != nil { + logger.Error("failed to encode checkpoint root", + "err", err, + ) + return err + } + + // Dump the write log. + for { + more, err := it.Next() + if err != nil { + logger.Error("failed to fetch next item from write log iterator", + "err", err, + ) + return err + } + + if !more { + return nil + } + + val, err := it.Value() + if err != nil { + logger.Error("failed to get value from write log iterator", + "err", err, + ) + return err + } + + if err = enc.Encode([][]byte{val.Key, val.Value}); err != nil { + logger.Error("failed to encode write log entry", + "err", err, + ) + return err + } + } +} + +func newDirectStorageBackend(dataDir string) (storageAPI.Backend, error) { + // The right thing to do will be to use storage.New, but the backend config + // assumes that identity is valid, and we don't have one. + cfg := &storageAPI.Config{ + Backend: strings.ToLower(viper.GetString(storage.CfgBackend)), + DB: dataDir, + ApplyLockLRUSlots: uint64(viper.GetInt(storage.CfgLRUSlots)), + } + + b := strings.ToLower(viper.GetString(storage.CfgBackend)) + switch b { + case storageDatabase.BackendNameBadgerDB: + cfg.DB = filepath.Join(cfg.DB, storageDatabase.DefaultFileName(cfg.Backend)) + return storageDatabase.New(cfg) + case storageClient.BackendName: + return storageClient.New(context.Background(), nil, nil, nil) + default: + return nil, fmt.Errorf("storage: unsupported backend: '%v'", cfg.Backend) + } +} + +func init() { + storageExportFlags.String(cfgExportDir, "", "the destination directory for storage dumps") + _ = viper.BindPFlags(storageExportFlags) +} diff --git a/go/oasis-node/cmd/debug/storage/storage.go b/go/oasis-node/cmd/debug/storage/storage.go index 6d6d1cb0b2f..032fd22ef26 100644 --- a/go/oasis-node/cmd/debug/storage/storage.go +++ b/go/oasis-node/cmd/debug/storage/storage.go @@ -17,6 +17,7 @@ import ( cmdControl "github.com/oasislabs/oasis-core/go/oasis-node/cmd/control" "github.com/oasislabs/oasis-core/go/roothash/api/block" runtimeClient "github.com/oasislabs/oasis-core/go/runtime/client/api" + "github.com/oasislabs/oasis-core/go/storage" storageAPI "github.com/oasislabs/oasis-core/go/storage/api" storageClient "github.com/oasislabs/oasis-core/go/storage/client" "github.com/oasislabs/oasis-core/go/storage/mkvs/urkel/node" @@ -292,7 +293,13 @@ func Register(parentCmd *cobra.Command) { storageForceFinalizeCmd.PersistentFlags().AddFlagSet(cmdGrpc.ClientFlags) storageForceFinalizeCmd.PersistentFlags().AddFlagSet(cmdFlags.DebugDontBlameOasisFlag) + storageExportCmd.Flags().AddFlagSet(storage.Flags) + storageExportCmd.Flags().AddFlagSet(cmdFlags.GenesisFileFlags) + storageExportCmd.Flags().AddFlagSet(cmdFlags.DebugDontBlameOasisFlag) + storageExportCmd.Flags().AddFlagSet(storageExportFlags) + storageCmd.AddCommand(storageCheckRootsCmd) storageCmd.AddCommand(storageForceFinalizeCmd) + storageCmd.AddCommand(storageExportCmd) parentCmd.AddCommand(storageCmd) } diff --git a/go/oasis-test-runner/cmd/root.go b/go/oasis-test-runner/cmd/root.go index fcf8b2dc703..f84bfe4e0cf 100644 --- a/go/oasis-test-runner/cmd/root.go +++ b/go/oasis-test-runner/cmd/root.go @@ -153,6 +153,13 @@ func runRoot(cmd *cobra.Command, args []string) error { } } + excludeMap := make(map[string]bool) + if excludeEnv := os.Getenv("OASIS_EXCLUDE_E2E"); excludeEnv != "" { + for _, v := range strings.Split(excludeEnv, ",") { + excludeMap[strings.ToLower(v)] = true + } + } + // Run the required test scenarios. parallelJobCount := viper.GetInt(cfgParallelJobCount) parallelJobIndex := viper.GetInt(cfgParallelJobIndex) @@ -167,6 +174,13 @@ func runRoot(cmd *cobra.Command, args []string) error { continue } + if excludeMap[strings.ToLower(n)] { + logger.Info("skipping test case (excluded by environment)", + "test", n, + ) + continue + } + logger.Info("running test case", "test", n, ) diff --git a/go/oasis-test-runner/scenario/e2e/dump_restore.go b/go/oasis-test-runner/scenario/e2e/dump_restore.go index 10111b8a54c..1d81beb7685 100644 --- a/go/oasis-test-runner/scenario/e2e/dump_restore.go +++ b/go/oasis-test-runner/scenario/e2e/dump_restore.go @@ -68,6 +68,21 @@ func (sc *dumpRestoreImpl) Run(childEnv *env.Env) error { // Stop the network. sc.logger.Info("stopping the network") sc.basicImpl.net.Stop() + + // Dump storage. + args = []string{ + "debug", "storage", "export", + "--genesis.file", dumpPath, + "--datadir", sc.basicImpl.net.StorageWorkers()[0].DataDir(), + "--storage.export.dir", filepath.Join(childEnv.Dir(), "storage_dumps"), + "--debug.dont_blame_oasis", + "--debug.allow_test_keys", + } + if err = runSubCommand(childEnv, "storage-dump", sc.basicImpl.net.Config().NodeBinary, args); err != nil { + return fmt.Errorf("scenario/e2e/dump_restore: failed to dump storage: %w", err) + } + + // Reset all the state back to the vanilla state. if err = sc.basicImpl.cleanTendermintStorage(childEnv); err != nil { return fmt.Errorf("scenario/e2e/dump_restore: failed to clean tendemint storage: %w", err) } diff --git a/go/storage/init.go b/go/storage/init.go index 70e21b250ff..b6b34c83c72 100644 --- a/go/storage/init.go +++ b/go/storage/init.go @@ -21,9 +21,10 @@ import ( const ( // CfgBackend configures the storage backend flag. - CfgBackend = "storage.backend" - cfgCrashEnabled = "storage.crash.enabled" - cfgLRUSlots = "storage.root_cache.apply_lock_lru_slots" + CfgBackend = "storage.backend" + cfgCrashEnabled = "storage.crash.enabled" + // CfgLRUSlots configures the LRU apply lock slots. + CfgLRUSlots = "storage.root_cache.apply_lock_lru_slots" cfgInsecureSkipChecks = "storage.debug.insecure_skip_checks" ) @@ -42,7 +43,7 @@ func New( Backend: strings.ToLower(viper.GetString(CfgBackend)), DB: dataDir, Signer: identity.NodeSigner, - ApplyLockLRUSlots: uint64(viper.GetInt(cfgLRUSlots)), + ApplyLockLRUSlots: uint64(viper.GetInt(CfgLRUSlots)), InsecureSkipChecks: viper.GetBool(cfgInsecureSkipChecks) && cmdFlags.DebugDontBlameOasis(), } @@ -75,7 +76,7 @@ func New( func init() { Flags.String(CfgBackend, database.BackendNameBadgerDB, "Storage backend") Flags.Bool(cfgCrashEnabled, false, "Enable the crashing storage wrapper") - Flags.Int(cfgLRUSlots, 1000, "How many LRU slots to use for Apply call locks in the MKVS tree root cache") + Flags.Int(CfgLRUSlots, 1000, "How many LRU slots to use for Apply call locks in the MKVS tree root cache") Flags.Bool(cfgInsecureSkipChecks, false, "INSECURE: Skip known root checks")