From 09ebd1a55774826578a5b5a03d3b96b17c5acecd Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 1 Sep 2020 16:38:24 -0700 Subject: [PATCH 1/4] allow exporting a number of recent chain state trees --- api/api_full.go | 5 ++++- api/apistruct/struct.go | 6 +++--- chain/store/store.go | 17 +++++++++++------ chain/store/store_test.go | 2 +- cli/chain.go | 8 +++++++- node/impl/full/chain.go | 4 ++-- 6 files changed, 28 insertions(+), 14 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 2f146267be6..ffceaccb636 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -109,7 +109,10 @@ type FullNode interface { ChainGetPath(ctx context.Context, from types.TipSetKey, to types.TipSetKey) ([]*HeadChange, error) // ChainExport returns a stream of bytes with CAR dump of chain data. - ChainExport(context.Context, types.TipSetKey) (<-chan []byte, error) + // The exported chain data includes the header chain from the given tipset + // back to genesis, the entire genesis state, and the most recent 'nroots' + // state trees. + ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) // MethodGroup: Beacon // The Beacon method group contains methods for interacting with the random beacon (DRAND) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index 33b02933fba..ad8c5d40fb3 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -86,7 +86,7 @@ type FullNodeStruct struct { ChainGetNode func(ctx context.Context, p string) (*api.IpldObject, error) `perm:"read"` ChainGetMessage func(context.Context, cid.Cid) (*types.Message, error) `perm:"read"` ChainGetPath func(context.Context, types.TipSetKey, types.TipSetKey) ([]*api.HeadChange, error) `perm:"read"` - ChainExport func(context.Context, types.TipSetKey) (<-chan []byte, error) `perm:"read"` + ChainExport func(context.Context, abi.ChainEpoch, types.TipSetKey) (<-chan []byte, error) `perm:"read"` BeaconGetEntry func(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) `perm:"read"` @@ -654,8 +654,8 @@ func (c *FullNodeStruct) ChainGetPath(ctx context.Context, from types.TipSetKey, return c.Internal.ChainGetPath(ctx, from, to) } -func (c *FullNodeStruct) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) { - return c.Internal.ChainExport(ctx, tsk) +func (c *FullNodeStruct) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) { + return c.Internal.ChainExport(ctx, nroots, tsk) } func (c *FullNodeStruct) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch) (*types.BeaconEntry, error) { diff --git a/chain/store/store.go b/chain/store/store.go index 01e8b6a7114..ac99b98c39d 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1114,7 +1114,7 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t return cs.LoadTipSet(lbts.Parents()) } -func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) { +func recurseLinks(bs bstore.Blockstore, seen *cid.Set, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) { if root.Prefix().Codec != cid.DagCBOR { return in, nil } @@ -1131,9 +1131,14 @@ func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, return } + // traversed this already... + if !seen.Visit(c) { + return + } + in = append(in, c) var err error - in, err = recurseLinks(bs, c, in) + in, err = recurseLinks(bs, seen, c, in) if err != nil { rerr = err } @@ -1145,7 +1150,7 @@ func recurseLinks(bs bstore.Blockstore, root cid.Cid, in []cid.Cid) ([]cid.Cid, return in, rerr } -func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer) error { +func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, w io.Writer) error { if ts == nil { ts = cs.GetHeaviestTipSet() } @@ -1182,7 +1187,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer) return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) } - cids, err := recurseLinks(cs.bs, b.Messages, []cid.Cid{b.Messages}) + cids, err := recurseLinks(cs.bs, seen, b.Messages, []cid.Cid{b.Messages}) if err != nil { return xerrors.Errorf("recursing messages failed: %w", err) } @@ -1198,8 +1203,8 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, w io.Writer) out := cids - if b.Height == 0 { - cids, err := recurseLinks(cs.bs, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot}) + if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots { + cids, err := recurseLinks(cs.bs, seen, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot}) if err != nil { return xerrors.Errorf("recursing genesis state failed: %w", err) } diff --git a/chain/store/store_test.go b/chain/store/store_test.go index ec47245d679..42de4c19d4e 100644 --- a/chain/store/store_test.go +++ b/chain/store/store_test.go @@ -96,7 +96,7 @@ func TestChainExportImport(t *testing.T) { } buf := new(bytes.Buffer) - if err := cg.ChainStore().Export(context.TODO(), last, buf); err != nil { + if err := cg.ChainStore().Export(context.TODO(), last, 0, buf); err != nil { t.Fatal(err) } diff --git a/cli/chain.go b/cli/chain.go index 1f8339e932d..ea41537339c 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -859,6 +859,10 @@ var chainExportCmd = &cli.Command{ &cli.StringFlag{ Name: "tipset", }, + &cli.Int64Flag{ + Name: "recent-stateroots", + Usage: "specify the number of recent state roots to include in the export", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -872,6 +876,8 @@ var chainExportCmd = &cli.Command{ return fmt.Errorf("must specify filename to export chain to") } + rsrs := cctx.Int64("recent-stateroots") + fi, err := os.Create(cctx.Args().First()) if err != nil { return err @@ -888,7 +894,7 @@ var chainExportCmd = &cli.Command{ return err } - stream, err := api.ChainExport(ctx, ts.Key()) + stream, err := api.ChainExport(ctx, abi.ChainEpoch(rsrs), ts.Key()) if err != nil { return err } diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index 7c254393e06..61eb4d3f572 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -494,7 +494,7 @@ func (a *ChainAPI) ChainGetMessage(ctx context.Context, mc cid.Cid) (*types.Mess return cm.VMMessage(), nil } -func (a *ChainAPI) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan []byte, error) { +func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, tsk types.TipSetKey) (<-chan []byte, error) { ts, err := a.Chain.GetTipSetFromKey(tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) @@ -503,7 +503,7 @@ func (a *ChainAPI) ChainExport(ctx context.Context, tsk types.TipSetKey) (<-chan out := make(chan []byte) go func() { defer w.Close() //nolint:errcheck // it is a pipe - if err := a.Chain.Export(ctx, ts, w); err != nil { + if err := a.Chain.Export(ctx, ts, nroots, w); err != nil { log.Errorf("chain export call failed: %s", err) return } From 2b16e69e90ad767eb9a482a160956369a6fbc609 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 1 Sep 2020 20:12:21 -0700 Subject: [PATCH 2/4] allow snapshot importing --- chain/store/store.go | 11 ++++++----- chain/store/weight.go | 2 +- cmd/lotus/daemon.go | 28 ++++++++++++++++++++++------ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/chain/store/store.go b/chain/store/store.go index ac99b98c39d..af78ff28634 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1114,7 +1114,7 @@ func (cs *ChainStore) GetTipsetByHeight(ctx context.Context, h abi.ChainEpoch, t return cs.LoadTipSet(lbts.Parents()) } -func recurseLinks(bs bstore.Blockstore, seen *cid.Set, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) { +func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid.Cid) ([]cid.Cid, error) { if root.Prefix().Codec != cid.DagCBOR { return in, nil } @@ -1132,13 +1132,13 @@ func recurseLinks(bs bstore.Blockstore, seen *cid.Set, root cid.Cid, in []cid.Ci } // traversed this already... - if !seen.Visit(c) { + if !walked.Visit(c) { return } in = append(in, c) var err error - in, err = recurseLinks(bs, seen, c, in) + in, err = recurseLinks(bs, walked, c, in) if err != nil { rerr = err } @@ -1156,6 +1156,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo } seen := cid.NewSet() + walked := cid.NewSet() h := &car.CarHeader{ Roots: ts.Cids(), @@ -1187,7 +1188,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) } - cids, err := recurseLinks(cs.bs, seen, b.Messages, []cid.Cid{b.Messages}) + cids, err := recurseLinks(cs.bs, walked, b.Messages, []cid.Cid{b.Messages}) if err != nil { return xerrors.Errorf("recursing messages failed: %w", err) } @@ -1204,7 +1205,7 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo out := cids if b.Height == 0 || b.Height > ts.Height()-inclRecentRoots { - cids, err := recurseLinks(cs.bs, seen, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot}) + cids, err := recurseLinks(cs.bs, walked, b.ParentStateRoot, []cid.Cid{b.ParentStateRoot}) if err != nil { return xerrors.Errorf("recursing genesis state failed: %w", err) } diff --git a/chain/store/weight.go b/chain/store/weight.go index 3fce45692e2..2e8516f57f8 100644 --- a/chain/store/weight.go +++ b/chain/store/weight.go @@ -41,7 +41,7 @@ func (cs *ChainStore) Weight(ctx context.Context, ts *types.TipSet) (types.BigIn var st power.State if err := cst.Get(ctx, act.Head, &st); err != nil { - return types.NewInt(0), xerrors.Errorf("get power actor head: %w", err) + return types.NewInt(0), xerrors.Errorf("get power actor head (%s, height=%d): %w", act.Head, ts.Height(), err) } tpow = st.TotalQualityAdjPower // TODO: REVIEW: Is this correct? } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index a99b0c104e5..f1567938b0c 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -102,6 +102,10 @@ var DaemonCmd = &cli.Command{ Name: "import-chain", Usage: "on first run, load chain from given file", }, + &cli.StringFlag{ + Name: "checkpoint", + Usage: "import chain state from a given chain export file", + }, &cli.BoolFlag{ Name: "halt-after-import", Usage: "halt the process after importing chain from file", @@ -191,13 +195,23 @@ var DaemonCmd = &cli.Command{ } chainfile := cctx.String("import-chain") - if chainfile != "" { + snapshot := cctx.String("checkpoint") + if chainfile != "" || snapshot != "" { + if chainfile != "" && snapshot != "" { + return fmt.Errorf("cannot specify both 'snapshot' and 'import-chain'") + } + var ischeckpoint bool + if chainfile == "" { + chainfile = snapshot + ischeckpoint = true + } + chainfile, err := homedir.Expand(chainfile) if err != nil { return err } - if err := ImportChain(r, chainfile); err != nil { + if err := ImportChain(r, chainfile, ischeckpoint); err != nil { return err } if cctx.Bool("halt-after-import") { @@ -312,7 +326,7 @@ func importKey(ctx context.Context, api api.FullNode, f string) error { return nil } -func ImportChain(r repo.Repo, fname string) error { +func ImportChain(r repo.Repo, fname string, snapshot bool) error { fi, err := os.Open(fname) if err != nil { return err @@ -357,9 +371,11 @@ func ImportChain(r repo.Repo, fname string) error { stm := stmgr.NewStateManager(cst) - log.Infof("validating imported chain...") - if err := stm.ValidateChain(context.TODO(), ts); err != nil { - return xerrors.Errorf("chain validation failed: %w", err) + if !snapshot { + log.Infof("validating imported chain...") + if err := stm.ValidateChain(context.TODO(), ts); err != nil { + return xerrors.Errorf("chain validation failed: %w", err) + } } log.Info("accepting %s as new head", ts.Cids()) From 528e39fcb323cd67fd782c3f728e1faa3c9703e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 2 Sep 2020 14:57:44 +0200 Subject: [PATCH 3/4] docsgen, fix snapshot flag name --- cmd/lotus/daemon.go | 10 +++++----- documentation/en/api-methods.md | 4 ++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index f1567938b0c..b7365662ea1 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -103,7 +103,7 @@ var DaemonCmd = &cli.Command{ Usage: "on first run, load chain from given file", }, &cli.StringFlag{ - Name: "checkpoint", + Name: "snapshot", Usage: "import chain state from a given chain export file", }, &cli.BoolFlag{ @@ -195,15 +195,15 @@ var DaemonCmd = &cli.Command{ } chainfile := cctx.String("import-chain") - snapshot := cctx.String("checkpoint") + snapshot := cctx.String("snapshot") if chainfile != "" || snapshot != "" { if chainfile != "" && snapshot != "" { return fmt.Errorf("cannot specify both 'snapshot' and 'import-chain'") } - var ischeckpoint bool + var issnapshot bool if chainfile == "" { chainfile = snapshot - ischeckpoint = true + issnapshot = true } chainfile, err := homedir.Expand(chainfile) @@ -211,7 +211,7 @@ var DaemonCmd = &cli.Command{ return err } - if err := ImportChain(r, chainfile, ischeckpoint); err != nil { + if err := ImportChain(r, chainfile, issnapshot); err != nil { return err } if cctx.Bool("halt-after-import") { diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index fbee92c8357..d60e87eb18c 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -268,6 +268,9 @@ blockchain, but that do not require any form of state computation. ### ChainExport ChainExport returns a stream of bytes with CAR dump of chain data. +The exported chain data includes the header chain from the given tipset +back to genesis, the entire genesis state, and the most recent 'nroots' +state trees. Perms: read @@ -275,6 +278,7 @@ Perms: read Inputs: ```json [ + 10101, [ { "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" From 59f765f7bec1732cc96e600bca945073dada8a36 Mon Sep 17 00:00:00 2001 From: Jakub Sztandera Date: Wed, 2 Sep 2020 15:47:18 +0200 Subject: [PATCH 4/4] Rename to import-snapshot, require more than finality for state export Signed-off-by: Jakub Sztandera --- cli/chain.go | 7 +++++-- cmd/lotus/daemon.go | 8 ++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cli/chain.go b/cli/chain.go index ea41537339c..1d203639a98 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -876,7 +876,10 @@ var chainExportCmd = &cli.Command{ return fmt.Errorf("must specify filename to export chain to") } - rsrs := cctx.Int64("recent-stateroots") + rsrs := abi.ChainEpoch(cctx.Int64("recent-stateroots")) + if cctx.IsSet("recent-stateroots") && rsrs < build.Finality { + return fmt.Errorf("\"recent-stateroots\" has to be greater than %d", build.Finality) + } fi, err := os.Create(cctx.Args().First()) if err != nil { @@ -894,7 +897,7 @@ var chainExportCmd = &cli.Command{ return err } - stream, err := api.ChainExport(ctx, abi.ChainEpoch(rsrs), ts.Key()) + stream, err := api.ChainExport(ctx, rsrs, ts.Key()) if err != nil { return err } diff --git a/cmd/lotus/daemon.go b/cmd/lotus/daemon.go index b7365662ea1..e0fee656463 100644 --- a/cmd/lotus/daemon.go +++ b/cmd/lotus/daemon.go @@ -100,10 +100,10 @@ var DaemonCmd = &cli.Command{ }, &cli.StringFlag{ Name: "import-chain", - Usage: "on first run, load chain from given file", + Usage: "on first run, load chain from given file and validate", }, &cli.StringFlag{ - Name: "snapshot", + Name: "import-snapshot", Usage: "import chain state from a given chain export file", }, &cli.BoolFlag{ @@ -195,10 +195,10 @@ var DaemonCmd = &cli.Command{ } chainfile := cctx.String("import-chain") - snapshot := cctx.String("snapshot") + snapshot := cctx.String("import-snapshot") if chainfile != "" || snapshot != "" { if chainfile != "" && snapshot != "" { - return fmt.Errorf("cannot specify both 'snapshot' and 'import-chain'") + return fmt.Errorf("cannot specify both 'import-snapshot' and 'import-chain'") } var issnapshot bool if chainfile == "" {