cmd: command to dump preimages enumerated in snapshot order, in a flat file

cmd/geth/chaincmd.go

@@ -144,6 +144,17 @@ It's deprecated, please use "geth db import" instead.
 		Description: `
 The export-preimages command exports hash preimages to an RLP encoded stream.
 It's deprecated, please use "geth db export" instead.
+`,
+	}
+	exportOverlayPreimagesCommand = &cli.Command{
+		Action:    exportOverlayPreimages,
+		Name:      "export-overlay-preimages",
+		Usage:     "Export the preimage in overlay tree migration order",
+		ArgsUsage: "<dumpfile>",
+		Flags:     flags.Merge([]cli.Flag{}, utils.DatabasePathFlags),
+		Description: `
+The export-overlay-preimages command exports hash preimages to a flat file, in exactly
+the expected order for the overlay tree migration.
 `,
 	}
 	dumpCommand = &cli.Command{
@@ -399,6 +410,24 @@ func exportPreimages(ctx *cli.Context) error {
 	return nil
 }
 
+// exportOverlayPreimages dumps the preimage data to a flat file.
+func exportOverlayPreimages(ctx *cli.Context) error {
+	if ctx.Args().Len() < 1 {
+		utils.Fatalf("This command requires an argument.")
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	chain, _ := utils.MakeChain(ctx, stack, true)
+
+	start := time.Now()
+	if err := utils.ExportOverlayPreimages(chain, ctx.Args().First()); err != nil {
+		utils.Fatalf("Export error: %v\n", err)
+	}
+	fmt.Printf("Export done in %v\n", time.Since(start))
+	return nil
+}
+
 func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
 	db := utils.MakeChainDatabase(ctx, stack, true)
 	var header *types.Header

cmd/geth/main.go

@@ -209,6 +209,7 @@ func init() {
 		exportCommand,
 		importPreimagesCommand,
 		exportPreimagesCommand,
+		exportOverlayPreimagesCommand,
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,

cmd/utils/cmd.go

@@ -374,6 +374,73 @@ func ExportPreimages(db ethdb.Database, fn string) error {
 	return nil
 }
 
+// ExportOverlayPreimages exports all known hash preimages into the specified file,
+// in the same order as expected by the overlay tree migration.
+func ExportOverlayPreimages(chain *core.BlockChain, fn string) error {
+	log.Info("Exporting preimages", "file", fn)
+
+	fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	defer fh.Close()
+
+	writer := bufio.NewWriter(fh)
+	defer writer.Flush()
+
+	statedb, err := chain.State()
+	if err != nil {
+		return fmt.Errorf("failed to open statedb: %w", err)
+	}
+
+	mptRoot := chain.CurrentBlock().Root
+
+	accIt, err := chain.Snapshots().AccountIterator(mptRoot, common.Hash{})
+	if err != nil {
+		return err
+	}
+	defer accIt.Release()
+
+	count := 0
+	for accIt.Next() {
+		acc, err := types.FullAccount(accIt.Account())
+		if err != nil {
+			return fmt.Errorf("invalid account encountered during traversal: %s", err)
+		}
+		addr := rawdb.ReadPreimage(statedb.Database().DiskDB(), accIt.Hash())
+		if len(addr) != 20 {
+			return fmt.Errorf("addr len is zero is not 32: %d", len(addr))
+		}
+		if _, err := writer.Write(addr); err != nil {
+			return fmt.Errorf("failed to write addr preimage: %w", err)
+		}
+
+		if acc.Root == types.EmptyRootHash {
+			stIt, err := chain.Snapshots().StorageIterator(mptRoot, accIt.Hash(), common.Hash{})
+			if err != nil {
+				return fmt.Errorf("failed to create storage iterator: %w", err)
+			}
+			for stIt.Next() {
+				slotnr := rawdb.ReadPreimage(statedb.Database().DiskDB(), stIt.Hash())
+				if len(slotnr) != 32 {
+					return fmt.Errorf("slotnr not 32 len")
+				}
+				if _, err := writer.Write(slotnr); err != nil {
+					return fmt.Errorf("failed to write slotnr preimage: %w", err)
+				}
+			}
+			stIt.Release()
+		}
+		count++
+		if count%100000 == 0 {
+			log.Info("Last exported account", "account", accIt.Hash())
+		}
+	}
+
+	log.Info("Exported preimages", "file", fn)
+	return nil
+}
+
 // exportHeader is used in the export/import flow. When we do an export,
 // the first element we output is the exportHeader.
 // Whenever a backwards-incompatible change is made, the Version header