From c610bd818ea3b73414d4521c10ac5e8333b8e447 Mon Sep 17 00:00:00 2001 From: jennijuju Date: Thu, 10 Sep 2020 14:34:18 -0400 Subject: [PATCH 01/27] Increased ExpectedSealDuration and and WaitDealsDelay. --- node/config/def.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/config/def.go b/node/config/def.go index 9fee8895af5..1e216238ee4 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -147,7 +147,7 @@ func DefaultStorageMiner() *StorageMiner { MaxWaitDealsSectors: 2, // 64G with 32G sectors MaxSealingSectors: 0, MaxSealingSectorsForDeals: 0, - WaitDealsDelay: Duration(time.Hour), + WaitDealsDelay: Duration(time.Hour * 6), }, Storage: sectorstorage.SealerConfig{ @@ -169,7 +169,7 @@ func DefaultStorageMiner() *StorageMiner { ConsiderOfflineRetrievalDeals: true, PieceCidBlocklist: []cid.Cid{}, // TODO: It'd be nice to set this based on sector size - ExpectedSealDuration: Duration(time.Hour * 12), + ExpectedSealDuration: Duration(time.Hour * 24), }, Fees: MinerFeeConfig{ From 6670d22fb5714f4dfc651323b75a03631b0ddf97 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 15 Sep 2020 20:20:48 -0700 Subject: [PATCH 02/27] add command to (slowly) prune lotus chain datastore --- chain/store/store.go | 46 +++--- cmd/lotus-shed/main.go | 1 + cmd/lotus-shed/pruning.go | 290 ++++++++++++++++++++++++++++++++++++++ go.mod | 1 + 4 files changed, 321 insertions(+), 17 deletions(-) create mode 100644 cmd/lotus-shed/pruning.go diff --git a/chain/store/store.go b/chain/store/store.go index 20a7e30311e..404befac7ea 100644 --- a/chain/store/store.go +++ b/chain/store/store.go @@ -1183,13 +1183,6 @@ func recurseLinks(bs bstore.Blockstore, walked *cid.Set, root cid.Cid, in []cid. } func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, w io.Writer) error { - if ts == nil { - ts = cs.GetHeaviestTipSet() - } - - seen := cid.NewSet() - walked := cid.NewSet() - h := &car.CarHeader{ Roots: ts.Cids(), Version: 1, @@ -1199,6 +1192,28 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return xerrors.Errorf("failed to write car header: %s", err) } + return cs.WalkSnapshot(ctx, ts, inclRecentRoots, skipOldMsgs, func(c cid.Cid) error { + blk, err := cs.bs.Get(c) + if err != nil { + return xerrors.Errorf("writing object to car, bs.Get: %w", err) + } + + if err := carutil.LdWrite(w, c.Bytes(), blk.RawData()); err != nil { + return xerrors.Errorf("failed to write block to car output: %w", err) + } + + return nil + }) +} + +func (cs *ChainStore) WalkSnapshot(ctx context.Context, ts *types.TipSet, inclRecentRoots abi.ChainEpoch, skipOldMsgs bool, cb func(cid.Cid) error) error { + if ts == nil { + ts = cs.GetHeaviestTipSet() + } + + seen := cid.NewSet() + walked := cid.NewSet() + blocksToWalk := ts.Cids() walkChain := func(blk cid.Cid) error { @@ -1206,15 +1221,15 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo return nil } + if err := cb(blk); err != nil { + return err + } + data, err := cs.bs.Get(blk) if err != nil { return xerrors.Errorf("getting block: %w", err) } - if err := carutil.LdWrite(w, blk.Bytes(), data.RawData()); err != nil { - return xerrors.Errorf("failed to write block to car output: %w", err) - } - var b types.BlockHeader if err := b.UnmarshalCBOR(bytes.NewBuffer(data.RawData())); err != nil { return xerrors.Errorf("unmarshaling block header (cid=%s): %w", blk, err) @@ -1254,14 +1269,11 @@ func (cs *ChainStore) Export(ctx context.Context, ts *types.TipSet, inclRecentRo if c.Prefix().Codec != cid.DagCBOR { continue } - data, err := cs.bs.Get(c) - if err != nil { - return xerrors.Errorf("writing object to car (get %s): %w", c, err) - } - if err := carutil.LdWrite(w, c.Bytes(), data.RawData()); err != nil { - return xerrors.Errorf("failed to write out car object: %w", err) + if err := cb(c); err != nil { + return err } + } } diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index cff3059b679..a64f981a172 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -35,6 +35,7 @@ func main() { mathCmd, mpoolStatsCmd, exportChainCmd, + stateTreePruneCmd, } app := &cli.App{ diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go new file mode 100644 index 00000000000..6f0c20541b5 --- /dev/null +++ b/cmd/lotus-shed/pruning.go @@ -0,0 +1,290 @@ +package main + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper" + "github.com/filecoin-project/lotus/lib/blockstore" + "github.com/filecoin-project/lotus/node/repo" + "github.com/ipfs/bbloom" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/query" + dshelp "github.com/ipfs/go-ipfs-ds-help" + "github.com/urfave/cli/v2" + "golang.org/x/xerrors" +) + +type cidSet interface { + Add(cid.Cid) + Has(cid.Cid) bool + HasRaw([]byte) bool + Len() int +} + +type bloomSet struct { + bloom *bbloom.Bloom +} + +func newBloomSet(size int64) (*bloomSet, error) { + b, err := bbloom.New(float64(size), 3) + if err != nil { + return nil, err + } + + return &bloomSet{bloom: b}, nil +} + +func (bs *bloomSet) Add(c cid.Cid) { + bs.bloom.Add(c.Hash()) + +} + +func (bs *bloomSet) Has(c cid.Cid) bool { + return bs.bloom.Has(c.Hash()) +} + +func (bs *bloomSet) HasRaw(b []byte) bool { + return bs.bloom.Has(b) +} + +func (bs *bloomSet) Len() int { + return int(bs.bloom.ElementsAdded()) +} + +type mapSet struct { + m map[string]struct{} +} + +func newMapSet() *mapSet { + return &mapSet{m: make(map[string]struct{})} +} + +func (bs *mapSet) Add(c cid.Cid) { + bs.m[string(c.Hash())] = struct{}{} +} + +func (bs *mapSet) Has(c cid.Cid) bool { + _, ok := bs.m[string(c.Hash())] + return ok +} + +func (bs *mapSet) HasRaw(b []byte) bool { + _, ok := bs.m[string(b)] + return ok +} + +func (bs *mapSet) Len() int { + return len(bs.m) +} + +var stateTreePruneCmd = &cli.Command{ + Name: "state-prune", + Description: "Deletes old state root data from local chainstore", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + }, + &cli.Int64Flag{ + Name: "keep-from-lookback", + Usage: "keep stateroots at or newer than the current height minus this lookback", + Value: 1800, // 2 x finality + }, + &cli.IntFlag{ + Name: "delete-up-to", + Usage: "delete up to the given number of objects (used to run a faster 'partial' sync)", + }, + &cli.BoolFlag{ + Name: "use-bloom-set", + Usage: "use a bloom filter for the 'good' set instead of a map, reduces memory usage but may not clean up as much", + }, + &cli.BoolFlag{ + Name: "dry-run", + Usage: "only enumerate the good set, don't do any deletions", + }, + &cli.BoolFlag{ + Name: "only-ds-gc", + Usage: "Only run datastore GC", + }, + &cli.IntFlag{ + Name: "gc-count", + Usage: "number of times to run gc", + Value: 20, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.TODO() + + fsrepo, err := repo.NewFS(cctx.String("repo")) + if err != nil { + return err + } + + lkrepo, err := fsrepo.Lock(repo.FullNode) + if err != nil { + return err + } + + defer lkrepo.Close() //nolint:errcheck + + ds, err := lkrepo.Datastore("/chain") + if err != nil { + return err + } + + defer ds.Close() + + mds, err := lkrepo.Datastore("/metadata") + if err != nil { + return err + } + defer mds.Close() + + if cctx.Bool("only-ds-gc") { + gcds, ok := ds.(datastore.GCDatastore) + if ok { + fmt.Println("running datastore gc....") + for i := 0; i < cctx.Int("gc-count"); i++ { + if err := gcds.CollectGarbage(); err != nil { + return xerrors.Errorf("datastore GC failed: %w", err) + } + } + fmt.Println("gc complete!") + return nil + } else { + return fmt.Errorf("datastore doesnt support gc") + } + } + + bs := blockstore.NewBlockstore(ds) + + cs := store.NewChainStore(bs, mds, vm.Syscalls(ffiwrapper.ProofVerifier)) + if err := cs.Load(); err != nil { + return fmt.Errorf("loading chainstore: %w", err) + } + + var goodSet cidSet + if cctx.Bool("use-bloom-set") { + bset, err := newBloomSet(10000000) + if err != nil { + return err + } + goodSet = bset + } else { + goodSet = newMapSet() + } + + ts := cs.GetHeaviestTipSet() + + rrLb := abi.ChainEpoch(cctx.Int64("keep-from-lookback")) + + if err := cs.WalkSnapshot(ctx, ts, rrLb, true, func(c cid.Cid) error { + if goodSet.Len()%20 == 0 { + fmt.Printf("\renumerating keep set: %d ", goodSet.Len()) + } + goodSet.Add(c) + return nil + }); err != nil { + return fmt.Errorf("snapshot walk failed: %w", err) + } + + fmt.Println() + fmt.Printf("Succesfully marked keep set! (%d objects)\n", goodSet.Len()) + + if cctx.Bool("dry-run") { + return nil + } + + var b datastore.Batch + var batchCount int + markForRemoval := func(c cid.Cid) error { + if b == nil { + nb, err := ds.Batch() + if err != nil { + return fmt.Errorf("opening batch: %w", err) + } + + b = nb + } + batchCount++ + + if err := b.Delete(dshelp.MultihashToDsKey(c.Hash())); err != nil { + return err + } + + if batchCount > 100 { + if err := b.Commit(); err != nil { + return xerrors.Errorf("failed to commit batch deletes: %w", err) + } + b = nil + batchCount = 0 + } + return nil + } + + res, err := ds.Query(query.Query{KeysOnly: true}) + if err != nil { + return xerrors.Errorf("failed to query datastore: %w", err) + } + + dupTo := cctx.Int("delete-up-to") + + var deleteCount int + var goodHits int + for { + v, ok := res.NextSync() + if !ok { + break + } + + bk, err := dshelp.BinaryFromDsKey(datastore.RawKey(v.Key[len("/blocks"):])) + if err != nil { + return xerrors.Errorf("failed to parse key: %w", err) + } + + if goodSet.HasRaw(bk) { + goodHits++ + continue + } + + nc := cid.NewCidV1(cid.Raw, bk) + + deleteCount++ + if err := markForRemoval(nc); err != nil { + return fmt.Errorf("failed to remove cid %s: %w", nc, err) + } + + if deleteCount%20 == 0 { + fmt.Printf("\rdeleting %d objects (good hits: %d)... ", deleteCount, goodHits) + } + + if dupTo != 0 && deleteCount > dupTo { + break + } + } + + if b != nil { + if err := b.Commit(); err != nil { + return xerrors.Errorf("failed to commit final batch delete: %w", err) + } + } + + gcds, ok := ds.(datastore.GCDatastore) + if ok { + fmt.Println("running datastore gc....") + for i := 0; i < cctx.Int("gc-count"); i++ { + if err := gcds.CollectGarbage(); err != nil { + return xerrors.Errorf("datastore GC failed: %w", err) + } + } + fmt.Println("gc complete!") + } + + return nil + }, +} diff --git a/go.mod b/go.mod index 65c2addb85b..b82f60dc8eb 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/golang-lru v0.5.4 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d + github.com/ipfs/bbloom v0.0.4 github.com/ipfs/go-bitswap v0.2.20 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.4-0.20200624145336-a978cec6e834 From 3c7246196933063fe6c01deb36cb650e96f2dead Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 18 Sep 2020 09:40:43 +0300 Subject: [PATCH 03/27] MpoolPushUntrusted API for gateway --- api/api_full.go | 3 ++ api/apistruct/struct.go | 8 +++- chain/messagepool/messagepool.go | 77 +++++++++++++++++++++++++++----- cmd/lotus-gateway/api.go | 2 +- node/impl/full/mpool.go | 4 ++ 5 files changed, 80 insertions(+), 14 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index dace85ed3cd..c60656e4d54 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -187,6 +187,9 @@ type FullNode interface { // MpoolPush pushes a signed message to mempool. MpoolPush(context.Context, *types.SignedMessage) (cid.Cid, error) + // MpoolPushUntrusted pushes a signed message to mempool from untrusted sources. + MpoolPushUntrusted(context.Context, *types.SignedMessage) (cid.Cid, error) + // MpoolPushMessage atomically assigns a nonce, signs, and pushes a message // to mempool. // maxFee is only used when GasFeeCap/GasPremium fields aren't specified diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index b37c667e998..f225ac5de5e 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -120,7 +120,9 @@ type FullNodeStruct struct { MpoolPending func(context.Context, types.TipSetKey) ([]*types.SignedMessage, error) `perm:"read"` MpoolClear func(context.Context, bool) error `perm:"write"` - MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"` + MpoolPush func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"` + MpoolPushUntrusted func(context.Context, *types.SignedMessage) (cid.Cid, error) `perm:"write"` + MpoolPushMessage func(context.Context, *types.Message, *api.MessageSendSpec) (*types.SignedMessage, error) `perm:"sign"` MpoolGetNonce func(context.Context, address.Address) (uint64, error) `perm:"read"` MpoolSub func(context.Context) (<-chan api.MpoolUpdate, error) `perm:"read"` @@ -549,6 +551,10 @@ func (c *FullNodeStruct) MpoolPush(ctx context.Context, smsg *types.SignedMessag return c.Internal.MpoolPush(ctx, smsg) } +func (c *FullNodeStruct) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return c.Internal.MpoolPushUntrusted(ctx, smsg) +} + func (c *FullNodeStruct) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) { return c.Internal.MpoolPushMessage(ctx, msg, spec) } diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index 96900925f1d..6ee86b73276 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -55,6 +55,7 @@ var baseFeeLowerBoundFactor = types.NewInt(10) var baseFeeLowerBoundFactorConservative = types.NewInt(100) var MaxActorPendingMessages = 1000 +var MaxUntrustedActorPendingMessages = 10 var MaxNonceGap = uint64(4) @@ -197,9 +198,17 @@ func CapGasFee(msg *types.Message, maxFee abi.TokenAmount) { msg.GasPremium = big.Min(msg.GasFeeCap, msg.GasPremium) // cap premium at FeeCap } -func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (bool, error) { +func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict, untrusted bool) (bool, error) { nextNonce := ms.nextNonce nonceGap := false + + maxNonceGap := MaxNonceGap + maxActorPendingMessages := MaxActorPendingMessages + if untrusted { + maxNonceGap = 0 + maxActorPendingMessages = MaxUntrustedActorPendingMessages + } + switch { case m.Message.Nonce == nextNonce: nextNonce++ @@ -208,7 +217,7 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (boo nextNonce++ } - case strict && m.Message.Nonce > nextNonce+MaxNonceGap: + case strict && m.Message.Nonce > nextNonce+maxNonceGap: return false, xerrors.Errorf("message nonce has too big a gap from expected nonce (Nonce: %d, nextNonce: %d): %w", m.Message.Nonce, nextNonce, ErrNonceGap) case m.Message.Nonce > nextNonce: @@ -244,7 +253,7 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict bool) (boo //ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.Value.Int) } - if !has && strict && len(ms.msgs) > MaxActorPendingMessages { + if !has && strict && len(ms.msgs) > maxActorPendingMessages { log.Errorf("too many pending messages from actor %s", m.Message.From) return false, ErrTooManyPendingMessages } @@ -486,7 +495,7 @@ func (mp *MessagePool) Push(m *types.SignedMessage) (cid.Cid, error) { } mp.curTsLk.Lock() - publish, err := mp.addTs(m, mp.curTs, true) + publish, err := mp.addTs(m, mp.curTs, true, false) if err != nil { mp.curTsLk.Unlock() return cid.Undef, err @@ -553,7 +562,7 @@ func (mp *MessagePool) Add(m *types.SignedMessage) error { mp.curTsLk.Lock() defer mp.curTsLk.Unlock() - _, err = mp.addTs(m, mp.curTs, false) + _, err = mp.addTs(m, mp.curTs, false, false) return err } @@ -621,7 +630,7 @@ func (mp *MessagePool) checkBalance(m *types.SignedMessage, curTs *types.TipSet) return nil } -func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local bool) (bool, error) { +func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local, untrusted bool) (bool, error) { snonce, err := mp.getStateNonce(m.Message.From, curTs) if err != nil { return false, xerrors.Errorf("failed to look up actor state nonce: %s: %w", err, ErrSoftValidationFailure) @@ -643,7 +652,7 @@ func (mp *MessagePool) addTs(m *types.SignedMessage, curTs *types.TipSet, local return false, err } - return publish, mp.addLocked(m, !local) + return publish, mp.addLocked(m, !local, untrusted) } func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { @@ -678,17 +687,17 @@ func (mp *MessagePool) addLoaded(m *types.SignedMessage) error { return err } - return mp.addLocked(m, false) + return mp.addLocked(m, false, false) } func (mp *MessagePool) addSkipChecks(m *types.SignedMessage) error { mp.lk.Lock() defer mp.lk.Unlock() - return mp.addLocked(m, false) + return mp.addLocked(m, false, false) } -func (mp *MessagePool) addLocked(m *types.SignedMessage, strict bool) error { +func (mp *MessagePool) addLocked(m *types.SignedMessage, strict, untrusted bool) error { log.Debugf("mpooladd: %s %d", m.Message.From, m.Message.Nonce) if m.Signature.Type == crypto.SigTypeBLS { mp.blsSigCache.Add(m.Cid(), m.Signature) @@ -715,7 +724,7 @@ func (mp *MessagePool) addLocked(m *types.SignedMessage, strict bool) error { mp.pending[m.Message.From] = mset } - incr, err := mset.add(m, mp, strict) + incr, err := mset.add(m, mp, strict, untrusted) if err != nil { log.Debug(err) return err @@ -873,7 +882,7 @@ func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, return nil, err } - if err := mp.addLocked(msg, false); err != nil { + if err := mp.addLocked(msg, false, false); err != nil { return nil, xerrors.Errorf("add locked failed: %w", err) } if err := mp.addLocal(msg, msgb); err != nil { @@ -887,6 +896,50 @@ func (mp *MessagePool) PushWithNonce(ctx context.Context, addr address.Address, return msg, err } +// this method is provided for the gateway to push messages. +// differences from Push: +// - strict checks are enabled +// - extra strict add checks are used when adding the messages to the msgSet +// that means: no nonce gaps, at most 10 pending messages for the actor +func (mp *MessagePool) PushUntrusted(m *types.SignedMessage) (cid.Cid, error) { + err := mp.checkMessage(m) + if err != nil { + return cid.Undef, err + } + + // serialize push access to reduce lock contention + mp.addSema <- struct{}{} + defer func() { + <-mp.addSema + }() + + msgb, err := m.Serialize() + if err != nil { + return cid.Undef, err + } + + mp.curTsLk.Lock() + publish, err := mp.addTs(m, mp.curTs, false, true) + if err != nil { + mp.curTsLk.Unlock() + return cid.Undef, err + } + mp.curTsLk.Unlock() + + mp.lk.Lock() + if err := mp.addLocal(m, msgb); err != nil { + mp.lk.Unlock() + return cid.Undef, err + } + mp.lk.Unlock() + + if publish { + err = mp.api.PubSubPublish(build.MessagesTopic(mp.netName), msgb) + } + + return m.Cid(), err +} + func (mp *MessagePool) Remove(from address.Address, nonce uint64, applied bool) { mp.lk.Lock() defer mp.lk.Unlock() diff --git a/cmd/lotus-gateway/api.go b/cmd/lotus-gateway/api.go index 42e9e482997..0a6365dbd08 100644 --- a/cmd/lotus-gateway/api.go +++ b/cmd/lotus-gateway/api.go @@ -86,5 +86,5 @@ func (a *GatewayAPI) MpoolPush(ctx context.Context, sm *types.SignedMessage) (ci // TODO: additional anti-spam checks - return a.api.MpoolPush(ctx, sm) + return a.api.MpoolPushUntrusted(ctx, sm) } diff --git a/node/impl/full/mpool.go b/node/impl/full/mpool.go index 6acb179902a..a847d8d8e41 100644 --- a/node/impl/full/mpool.go +++ b/node/impl/full/mpool.go @@ -113,6 +113,10 @@ func (a *MpoolAPI) MpoolPush(ctx context.Context, smsg *types.SignedMessage) (ci return a.Mpool.Push(smsg) } +func (a *MpoolAPI) MpoolPushUntrusted(ctx context.Context, smsg *types.SignedMessage) (cid.Cid, error) { + return a.Mpool.PushUntrusted(smsg) +} + func (a *MpoolAPI) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) { inMsg := *msg { From 26de0db75756b85f5135017fa43282b21b3255eb Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 9 Sep 2020 18:45:09 +0000 Subject: [PATCH 04/27] lotus-pcr: add recover-miners command --- cmd/lotus-pcr/main.go | 167 ++++++++++++++++++++++++++++++++++++++--- tools/stats/metrics.go | 22 ++++-- 2 files changed, 172 insertions(+), 17 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index dc12693cafe..473fc7ff13f 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -38,6 +38,7 @@ var log = logging.Logger("main") func main() { local := []*cli.Command{ runCmd, + recoverMinersCmd, versionCmd, } @@ -101,6 +102,80 @@ var versionCmd = &cli.Command{ }, } +var recoverMinersCmd = &cli.Command{ + Name: "recover-miners", + Usage: "Ensure all miners with a negative available balance have a FIL surplus across accounts", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "from", + EnvVars: []string{"LOTUS_PCR_FROM"}, + Usage: "wallet address to send refund from", + }, + &cli.BoolFlag{ + Name: "no-sync", + EnvVars: []string{"LOTUS_PCR_NO_SYNC"}, + Usage: "do not wait for chain sync to complete", + }, + &cli.BoolFlag{ + Name: "dry-run", + EnvVars: []string{"LOTUS_PCR_DRY_RUN"}, + Usage: "do not send any messages", + Value: false, + }, + &cli.IntFlag{ + Name: "miner-total-funds-threashold", + EnvVars: []string{"LOTUS_PCR_MINER_TOTAL_FUNDS_THREASHOLD"}, + Usage: "total filecoin across all accounts that should be met, if the miner balancer drops below zero", + Value: 0, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path")) + if err != nil { + log.Fatal(err) + } + defer closer() + + from, err := address.NewFromString(cctx.String("from")) + if err != nil { + return xerrors.Errorf("parsing source address (provide correct --from flag!): %w", err) + } + + if !cctx.Bool("no-sync") { + if err := stats.WaitForSyncComplete(ctx, api); err != nil { + log.Fatal(err) + } + } + + dryRun := cctx.Bool("dry-run") + minerTotalFundsThreashold := uint64(cctx.Int("miner-total-funds-threashold")) + + rf := &refunder{ + api: api, + wallet: from, + dryRun: dryRun, + minerTotalFundsThreashold: types.FromFil(minerTotalFundsThreashold), + } + + refundTipset, err := api.ChainHead(ctx) + if err != nil { + return err + } + + negativeBalancerRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund()) + if err != nil { + return err + } + + if err := rf.Refund(ctx, "refund negative balancer miner", refundTipset, negativeBalancerRefund, 0); err != nil { + return err + } + + return nil + }, +} + var runCmd = &cli.Command{ Name: "run", Usage: "Start message reimpursement", @@ -230,7 +305,7 @@ var runCmd = &cli.Command{ return err } - if err := rf.Refund(ctx, refundTipset, refunds, rounds); err != nil { + if err := rf.Refund(ctx, "refund stats", refundTipset, refunds, rounds); err != nil { return err } @@ -289,7 +364,6 @@ func (m *MinersRefund) Track(addr address.Address, value types.BigInt) { m.count = m.count + 1 m.totalRefunds = types.BigAdd(m.totalRefunds, value) - m.refunds[addr] = types.BigAdd(m.refunds[addr], value) } @@ -318,8 +392,12 @@ type refunderNodeApi interface { ChainGetParentMessages(ctx context.Context, blockCid cid.Cid) ([]api.Message, error) ChainGetParentReceipts(ctx context.Context, blockCid cid.Cid) ([]*types.MessageReceipt, error) ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) + ChainReadObj(context.Context, cid.Cid) ([]byte, error) StateMinerInitialPledgeCollateral(ctx context.Context, addr address.Address, precommitInfo miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) + StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) GasEstimateGasPremium(ctx context.Context, nblocksincl uint64, sender address.Address, gaslimit int64, tsk types.TipSetKey) (types.BigInt, error) @@ -327,12 +405,81 @@ type refunderNodeApi interface { } type refunder struct { - api refunderNodeApi - wallet address.Address - percentExtra int - dryRun bool - preCommitEnabled bool - proveCommitEnabled bool + api refunderNodeApi + wallet address.Address + percentExtra int + dryRun bool + preCommitEnabled bool + proveCommitEnabled bool + minerTotalFundsThreashold big.Int +} + +func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { + miners, err := r.api.StateListMiners(ctx, tipset.Key()) + if err != nil { + return nil, err + } + + for _, maddr := range miners { + mact, err := r.api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + if !mact.Balance.GreaterThan(big.Zero()) { + continue + } + + minerAvailableBalance, err := r.api.StateMinerAvailableBalance(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + if minerAvailableBalance.GreaterThanEqual(big.Zero()) { + log.Debugw("skipping over miner with positive balance", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + // Look up and find all addresses associated with the miner + minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key()) + + allAddresses := []address.Address{minerInfo.Worker, minerInfo.Owner} + allAddresses = append(allAddresses, minerInfo.ControlAddresses...) + + // Sum the balancer of all the addresses + addrSum := big.Zero() + addrCheck := make(map[address.Address]struct{}, len(allAddresses)) + for _, addr := range allAddresses { + if _, found := addrCheck[addr]; !found { + balance, err := r.api.WalletBalance(ctx, addr) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + addrSum = big.Add(addrSum, balance) + addrCheck[addr] = struct{}{} + } + } + + totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) + + // If the miner has available balance they should use it + if totalAvailableBalance.GreaterThanEqual(r.minerTotalFundsThreashold) { + log.Debugw("skipping over miner with enough funds cross all accounts", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "miner_available_balance", minerAvailableBalance, "wallet_total_balances", addrSum) + continue + } + + // Calculate the required FIL to bring the miner up to the minimum across all of their accounts + refundValue := big.Add(totalAvailableBalance.Abs(), r.minerTotalFundsThreashold) + refunds.Track(maddr, refundValue) + + log.Debugw("processing negative balance miner", "miner", maddr, "refund", refundValue) + } + + return refunds, nil } func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { @@ -464,7 +611,7 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu return refunds, nil } -func (r *refunder) Refund(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, rounds int) error { +func (r *refunder) Refund(ctx context.Context, name string, tipset *types.TipSet, refunds *MinersRefund, rounds int) error { if refunds.Count() == 0 { log.Debugw("no messages to refund in tipset", "height", tipset.Height(), "key", tipset.Key()) return nil @@ -522,7 +669,7 @@ func (r *refunder) Refund(ctx context.Context, tipset *types.TipSet, refunds *Mi refundSum = types.BigAdd(refundSum, msg.Value) } - log.Infow("refund stats", "tipsets_processed", rounds, "height", tipset.Height(), "key", tipset.Key(), "messages_sent", len(messages)-failures, "refund_sum", refundSum, "messages_failures", failures, "messages_processed", refunds.Count()) + log.Infow(name, "tipsets_processed", rounds, "height", tipset.Height(), "key", tipset.Key(), "messages_sent", len(messages)-failures, "refund_sum", refundSum, "messages_failures", failures, "messages_processed", refunds.Count()) return nil } diff --git a/tools/stats/metrics.go b/tools/stats/metrics.go index e50ac953f4f..eeef72e8e82 100644 --- a/tools/stats/metrics.go +++ b/tools/stats/metrics.go @@ -201,16 +201,24 @@ func RecordTipsetPoints(ctx context.Context, api api.FullNode, pl *PointList, ti return nil } -type apiIpldStore struct { +type ApiIpldStore struct { ctx context.Context - api api.FullNode + api apiIpldStoreApi } -func (ht *apiIpldStore) Context() context.Context { +type apiIpldStoreApi interface { + ChainReadObj(context.Context, cid.Cid) ([]byte, error) +} + +func NewApiIpldStore(ctx context.Context, api apiIpldStoreApi) *ApiIpldStore { + return &ApiIpldStore{ctx, api} +} + +func (ht *ApiIpldStore) Context() context.Context { return ht.ctx } -func (ht *apiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error { +func (ht *ApiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) error { raw, err := ht.api.ChainReadObj(ctx, c) if err != nil { return err @@ -227,8 +235,8 @@ func (ht *apiIpldStore) Get(ctx context.Context, c cid.Cid, out interface{}) err return fmt.Errorf("Object does not implement CBORUnmarshaler") } -func (ht *apiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) { - return cid.Undef, fmt.Errorf("Put is not implemented on apiIpldStore") +func (ht *ApiIpldStore) Put(ctx context.Context, v interface{}) (cid.Cid, error) { + return cid.Undef, fmt.Errorf("Put is not implemented on ApiIpldStore") } func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointList, tipset *types.TipSet) error { @@ -279,7 +287,7 @@ func RecordTipsetStatePoints(ctx context.Context, api api.FullNode, pl *PointLis return fmt.Errorf("failed to unmarshal power actor state: %w", err) } - s := &apiIpldStore{ctx, api} + s := NewApiIpldStore(ctx, api) mp, err := adt.AsMap(s, powerActorState.Claims) if err != nil { return err From e39036dd499ccc80697a2c38444953d77a1d7777 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Thu, 10 Sep 2020 07:36:24 +0000 Subject: [PATCH 05/27] lotus-pcr: find miners command --- cmd/lotus-pcr/main.go | 252 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 225 insertions(+), 27 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 473fc7ff13f..66279c29a0f 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -1,10 +1,13 @@ package main import ( + "bufio" "bytes" "context" + "encoding/csv" "fmt" "io/ioutil" + "math" "net/http" _ "net/http/pprof" "os" @@ -20,13 +23,14 @@ import ( "golang.org/x/xerrors" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-bitfield" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/types" @@ -39,6 +43,7 @@ func main() { local := []*cli.Command{ runCmd, recoverMinersCmd, + findMinersCmd, versionCmd, } @@ -102,6 +107,92 @@ var versionCmd = &cli.Command{ }, } +var findMinersCmd = &cli.Command{ + Name: "find-miners", + Usage: "find miners with a desired minimum balance", + Description: `Find miners returns a list of miners and their balances that are below a + threhold value. By default only the miner actor available balance is considered but other + account balances can be included by enabling them through the flags. + + Examples + Find all miners with an available balance below 100 FIL + + lotus-pcr find-miners --threshold 100 + + Find all miners with a balance below zero, which includes the owner and worker balances + + lotus-pcr find-miners --threshold 0 --owner --worker +`, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "no-sync", + EnvVars: []string{"LOTUS_PCR_NO_SYNC"}, + Usage: "do not wait for chain sync to complete", + }, + &cli.IntFlag{ + Name: "threshold", + EnvVars: []string{"LOTUS_PCR_THRESHOLD"}, + Usage: "balance below this limit will be printed", + Value: 0, + }, + &cli.BoolFlag{ + Name: "owner", + Usage: "include owner balance", + Value: false, + }, + &cli.BoolFlag{ + Name: "worker", + Usage: "include worker balance", + Value: false, + }, + &cli.BoolFlag{ + Name: "control", + Usage: "include control balance", + Value: false, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := context.Background() + api, closer, err := stats.GetFullNodeAPI(cctx.Context, cctx.String("lotus-path")) + if err != nil { + log.Fatal(err) + } + defer closer() + + if !cctx.Bool("no-sync") { + if err := stats.WaitForSyncComplete(ctx, api); err != nil { + log.Fatal(err) + } + } + + owner := cctx.Bool("owner") + worker := cctx.Bool("worker") + control := cctx.Bool("control") + threshold := uint64(cctx.Int("threshold")) + + rf := &refunder{ + api: api, + threshold: types.FromFil(threshold), + } + + refundTipset, err := api.ChainHead(ctx) + if err != nil { + return err + } + + balanceRefund, err := rf.FindMiners(ctx, refundTipset, NewMinersRefund(), owner, worker, control) + if err != nil { + return err + } + + for _, maddr := range balanceRefund.Miners() { + fmt.Printf("%s\t%s\n", maddr, types.FIL(balanceRefund.GetRefund(maddr))) + } + + return nil + }, +} + var recoverMinersCmd = &cli.Command{ Name: "recover-miners", Usage: "Ensure all miners with a negative available balance have a FIL surplus across accounts", @@ -123,11 +214,15 @@ var recoverMinersCmd = &cli.Command{ Value: false, }, &cli.IntFlag{ - Name: "miner-total-funds-threashold", - EnvVars: []string{"LOTUS_PCR_MINER_TOTAL_FUNDS_THREASHOLD"}, - Usage: "total filecoin across all accounts that should be met, if the miner balancer drops below zero", + Name: "threshold", + EnvVars: []string{"LOTUS_PCR_THRESHOLD"}, + Usage: "total filecoin across all accounts that should be met, if the miner balance drops below zero", Value: 0, }, + &cli.StringFlag{ + Name: "output", + Usage: "dump data as a csv format to this file", + }, }, Action: func(cctx *cli.Context) error { ctx := context.Background() @@ -149,13 +244,13 @@ var recoverMinersCmd = &cli.Command{ } dryRun := cctx.Bool("dry-run") - minerTotalFundsThreashold := uint64(cctx.Int("miner-total-funds-threashold")) + threshold := uint64(cctx.Int("threshold")) rf := &refunder{ - api: api, - wallet: from, - dryRun: dryRun, - minerTotalFundsThreashold: types.FromFil(minerTotalFundsThreashold), + api: api, + wallet: from, + dryRun: dryRun, + threshold: types.FromFil(threshold), } refundTipset, err := api.ChainHead(ctx) @@ -163,12 +258,12 @@ var recoverMinersCmd = &cli.Command{ return err } - negativeBalancerRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund()) + balanceRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund(), cctx.String("output")) if err != nil { return err } - if err := rf.Refund(ctx, "refund negative balancer miner", refundTipset, negativeBalancerRefund, 0); err != nil { + if err := rf.Refund(ctx, "refund to recover miner", refundTipset, balanceRefund, 0); err != nil { return err } @@ -397,6 +492,8 @@ type refunderNodeApi interface { StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) + StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) + StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error) StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) MpoolPushMessage(ctx context.Context, msg *types.Message, spec *api.MessageSendSpec) (*types.SignedMessage, error) @@ -405,16 +502,16 @@ type refunderNodeApi interface { } type refunder struct { - api refunderNodeApi - wallet address.Address - percentExtra int - dryRun bool - preCommitEnabled bool - proveCommitEnabled bool - minerTotalFundsThreashold big.Int + api refunderNodeApi + wallet address.Address + percentExtra int + dryRun bool + preCommitEnabled bool + proveCommitEnabled bool + threshold big.Int } -func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund) (*MinersRefund, error) { +func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) { miners, err := r.api.StateListMiners(ctx, tipset.Key()) if err != nil { return nil, err @@ -437,8 +534,91 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet continue } - if minerAvailableBalance.GreaterThanEqual(big.Zero()) { - log.Debugw("skipping over miner with positive balance", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + // Look up and find all addresses associated with the miner + minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key()) + + allAddresses := []address.Address{} + + if worker { + allAddresses = append(allAddresses, minerInfo.Worker) + } + + if owner { + allAddresses = append(allAddresses, minerInfo.Owner) + } + + if control { + allAddresses = append(allAddresses, minerInfo.ControlAddresses...) + } + + // Sum the balancer of all the addresses + addrSum := big.Zero() + addrCheck := make(map[address.Address]struct{}, len(allAddresses)) + for _, addr := range allAddresses { + if _, found := addrCheck[addr]; !found { + balance, err := r.api.WalletBalance(ctx, addr) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + addrSum = big.Add(addrSum, balance) + addrCheck[addr] = struct{}{} + } + } + + totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) + + if totalAvailableBalance.GreaterThanEqual(r.threshold) { + continue + } + + refunds.Track(maddr, totalAvailableBalance) + + log.Debugw("processing miner", "miner", maddr, "sectors", "available_balance", totalAvailableBalance) + } + + return refunds, nil +} + +func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, output string) (*MinersRefund, error) { + miners, err := r.api.StateListMiners(ctx, tipset.Key()) + if err != nil { + return nil, err + } + + w := ioutil.Discard + if len(output) != 0 { + f, err := os.Create(output) + if err != nil { + return nil, err + } + + defer f.Close() + + w = bufio.NewWriter(f) + } + + csvOut := csv.NewWriter(w) + defer csvOut.Flush() + if err := csvOut.Write([]string{"MinerID", "Sectors", "CombinedBalance", "ProposedSend"}); err != nil { + return nil, err + } + + for _, maddr := range miners { + mact, err := r.api.StateGetActor(ctx, maddr, types.EmptyTSK) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + if !mact.Balance.GreaterThan(big.Zero()) { + continue + } + + minerAvailableBalance, err := r.api.StateMinerAvailableBalance(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) continue } @@ -464,19 +644,37 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet } } + sectorInfo, err := r.api.StateMinerSectors(ctx, maddr, nil, false, tipset.Key()) + if err != nil { + log.Errorw("failed to look up miner sectors", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + if len(sectorInfo) == 0 { + log.Debugw("skipping miner with zero sectors", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) - // If the miner has available balance they should use it - if totalAvailableBalance.GreaterThanEqual(r.minerTotalFundsThreashold) { - log.Debugw("skipping over miner with enough funds cross all accounts", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "miner_available_balance", minerAvailableBalance, "wallet_total_balances", addrSum) + numSectorInfo := float64(len(sectorInfo)) + filAmount := uint64(math.Ceil(math.Max(math.Log(numSectorInfo)*math.Sqrt(numSectorInfo)/4, 20))) + attoFilAmount := big.Mul(big.NewIntUnsigned(filAmount), big.NewIntUnsigned(build.FilecoinPrecision)) + + if totalAvailableBalance.GreaterThanEqual(attoFilAmount) { + log.Debugw("skipping over miner with total available balance larger than refund", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "available_balance", totalAvailableBalance, "possible_refund", attoFilAmount) continue } - // Calculate the required FIL to bring the miner up to the minimum across all of their accounts - refundValue := big.Add(totalAvailableBalance.Abs(), r.minerTotalFundsThreashold) + refundValue := big.Sub(attoFilAmount, totalAvailableBalance) + refunds.Track(maddr, refundValue) + record := []string{maddr.String(), fmt.Sprintf("%d", len(sectorInfo)), totalAvailableBalance.String(), refundValue.String()} + if err := csvOut.Write(record); err != nil { + return nil, err + } - log.Debugw("processing negative balance miner", "miner", maddr, "refund", refundValue) + log.Debugw("processing miner", "miner", maddr, "sectors", len(sectorInfo), "available_balance", totalAvailableBalance, "refund", refundValue) } return refunds, nil From e6bbc03ca8e4207907d8fb44df3818b074815c48 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Sat, 19 Sep 2020 00:51:49 +0000 Subject: [PATCH 06/27] lotus-pcr: update miner recovery --- cmd/lotus-pcr/main.go | 343 +++++++++++++++++++++++++++++++++--------- 1 file changed, 269 insertions(+), 74 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 66279c29a0f..c80bbad4839 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -7,7 +7,6 @@ import ( "encoding/csv" "fmt" "io/ioutil" - "math" "net/http" _ "net/http/pprof" "os" @@ -213,16 +212,28 @@ var recoverMinersCmd = &cli.Command{ Usage: "do not send any messages", Value: false, }, - &cli.IntFlag{ - Name: "threshold", - EnvVars: []string{"LOTUS_PCR_THRESHOLD"}, - Usage: "total filecoin across all accounts that should be met, if the miner balance drops below zero", - Value: 0, - }, &cli.StringFlag{ Name: "output", Usage: "dump data as a csv format to this file", }, + &cli.IntFlag{ + Name: "miner-recovery-cutoff", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_CUTOFF"}, + Usage: "maximum amount of FIL that can be sent to any one miner before refund percent is applied", + Value: 3000, + }, + &cli.IntFlag{ + Name: "miner-recovery-bonus", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_BONUS"}, + Usage: "additional FIL to send to each miner", + Value: 5, + }, + &cli.IntFlag{ + Name: "miner-recovery-refund-percent", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_REFUND_PERCENT"}, + Usage: "percent of refund to issue", + Value: 110, + }, }, Action: func(cctx *cli.Context) error { ctx := context.Background() @@ -244,13 +255,17 @@ var recoverMinersCmd = &cli.Command{ } dryRun := cctx.Bool("dry-run") - threshold := uint64(cctx.Int("threshold")) + minerRecoveryRefundPercent := cctx.Int("miner-recovery-refund-percent") + minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff")) + minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus")) rf := &refunder{ - api: api, - wallet: from, - dryRun: dryRun, - threshold: types.FromFil(threshold), + api: api, + wallet: from, + dryRun: dryRun, + minerRecoveryRefundPercent: minerRecoveryRefundPercent, + minerRecoveryCutoff: types.FromFil(minerRecoveryCutoff), + minerRecoveryBonus: types.FromFil(minerRecoveryBonus), } refundTipset, err := api.ChainHead(ctx) @@ -286,10 +301,10 @@ var runCmd = &cli.Command{ Usage: "do not wait for chain sync to complete", }, &cli.IntFlag{ - Name: "percent-extra", - EnvVars: []string{"LOTUS_PCR_PERCENT_EXTRA"}, - Usage: "extra funds to send above the refund", - Value: 3, + Name: "refund-percent", + EnvVars: []string{"LOTUS_PCR_REFUND_PERCENT"}, + Usage: "percent of refund to issue", + Value: 103, }, &cli.IntFlag{ Name: "max-message-queue", @@ -327,6 +342,36 @@ var runCmd = &cli.Command{ Usage: "the number of tipsets to delay message processing to smooth chain reorgs", Value: int(build.MessageConfidence), }, + &cli.BoolFlag{ + Name: "miner-recovery", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY"}, + Usage: "run the miner recovery job", + Value: false, + }, + &cli.IntFlag{ + Name: "miner-recovery-period", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_PERIOD"}, + Usage: "interval between running miner recovery", + Value: 2880, + }, + &cli.IntFlag{ + Name: "miner-recovery-cutoff", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_CUTOFF"}, + Usage: "maximum amount of FIL that can be sent to any one miner before refund percent is applied", + Value: 3000, + }, + &cli.IntFlag{ + Name: "miner-recovery-bonus", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_BONUS"}, + Usage: "additional FIL to send to each miner", + Value: 5, + }, + &cli.IntFlag{ + Name: "miner-recovery-refund-percent", + EnvVars: []string{"LOTUS_PCR_MINER_RECOVERY_REFUND_PERCENT"}, + Usage: "percent of refund to issue", + Value: 110, + }, }, Action: func(cctx *cli.Context) error { go func() { @@ -365,24 +410,33 @@ var runCmd = &cli.Command{ log.Fatal(err) } - percentExtra := cctx.Int("percent-extra") + refundPercent := cctx.Int("refund-percent") maxMessageQueue := cctx.Int("max-message-queue") dryRun := cctx.Bool("dry-run") preCommitEnabled := cctx.Bool("pre-commit") proveCommitEnabled := cctx.Bool("prove-commit") aggregateTipsets := cctx.Int("aggregate-tipsets") + minerRecoveryEnabled := cctx.Bool("miner-recovery") + minerRecoveryPeriod := abi.ChainEpoch(int64(cctx.Int("miner-recovery-period"))) + minerRecoveryRefundPercent := cctx.Int("miner-recovery-refund-percent") + minerRecoveryCutoff := uint64(cctx.Int("miner-recovery-cutoff")) + minerRecoveryBonus := uint64(cctx.Int("miner-recovery-bonus")) rf := &refunder{ - api: api, - wallet: from, - percentExtra: percentExtra, - dryRun: dryRun, - preCommitEnabled: preCommitEnabled, - proveCommitEnabled: proveCommitEnabled, + api: api, + wallet: from, + refundPercent: refundPercent, + minerRecoveryRefundPercent: minerRecoveryRefundPercent, + minerRecoveryCutoff: types.FromFil(minerRecoveryCutoff), + minerRecoveryBonus: types.FromFil(minerRecoveryBonus), + dryRun: dryRun, + preCommitEnabled: preCommitEnabled, + proveCommitEnabled: proveCommitEnabled, } var refunds *MinersRefund = NewMinersRefund() var rounds int = 0 + nextMinerRecovery := r.MinerRecoveryHeight() + minerRecoveryPeriod for tipset := range tipsetsCh { refunds, err = rf.ProcessTipset(ctx, tipset, refunds) @@ -390,16 +444,33 @@ var runCmd = &cli.Command{ return err } - rounds = rounds + 1 - if rounds < aggregateTipsets { - continue - } - refundTipset, err := api.ChainHead(ctx) if err != nil { return err } + if minerRecoveryEnabled && refundTipset.Height() >= nextMinerRecovery { + recoveryRefund, err := rf.EnsureMinerMinimums(ctx, refundTipset, NewMinersRefund(), "") + if err != nil { + return err + } + + if err := rf.Refund(ctx, "refund to recover miners", refundTipset, recoveryRefund, 0); err != nil { + return err + } + + if err := r.SetMinerRecoveryHeight(tipset.Height()); err != nil { + return err + } + + nextMinerRecovery = r.MinerRecoveryHeight() + minerRecoveryPeriod + } + + rounds = rounds + 1 + if rounds < aggregateTipsets { + continue + } + if err := rf.Refund(ctx, "refund stats", refundTipset, refunds, rounds); err != nil { return err } @@ -502,13 +573,16 @@ type refunderNodeApi interface { } type refunder struct { - api refunderNodeApi - wallet address.Address - percentExtra int - dryRun bool - preCommitEnabled bool - proveCommitEnabled bool - threshold big.Int + api refunderNodeApi + wallet address.Address + refundPercent int + minerRecoveryRefundPercent int + minerRecoveryCutoff big.Int + minerRecoveryBonus big.Int + dryRun bool + preCommitEnabled bool + proveCommitEnabled bool + threshold big.Int } func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds *MinersRefund, owner, worker, control bool) (*MinersRefund, error) { @@ -601,7 +675,7 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet csvOut := csv.NewWriter(w) defer csvOut.Flush() - if err := csvOut.Write([]string{"MinerID", "Sectors", "CombinedBalance", "ProposedSend"}); err != nil { + if err := csvOut.Write([]string{"MinerID", "FaultedSectors", "AvailableBalance", "ProposedRefund"}); err != nil { return nil, err } @@ -644,37 +718,85 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet } } - sectorInfo, err := r.api.StateMinerSectors(ctx, maddr, nil, false, tipset.Key()) + faults, err := r.api.StateMinerFaults(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed to look up miner faults", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } + + faultsCount, err := faults.Count() if err != nil { - log.Errorw("failed to look up miner sectors", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + log.Errorw("failed to get count of faults", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) continue } - if len(sectorInfo) == 0 { - log.Debugw("skipping miner with zero sectors", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + if faultsCount == 0 { + log.Debugw("skipping miner with zero faults", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) continue } totalAvailableBalance := big.Add(addrSum, minerAvailableBalance) + balanceCutoff := big.Mul(big.Div(big.NewIntUnsigned(faultsCount), big.NewInt(10)), big.NewIntUnsigned(build.FilecoinPrecision)) + + if totalAvailableBalance.GreaterThan(balanceCutoff) { + log.Debugw( + "skipping over miner with total available balance larger than refund", + "height", tipset.Height(), + "key", tipset.Key(), + "miner", maddr, + "available_balance", totalAvailableBalance, + "balance_cutoff", balanceCutoff, + "faults_count", faultsCount, + "available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "balance_cutoff_fil", big.Div(balanceCutoff, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + ) + continue + } - numSectorInfo := float64(len(sectorInfo)) - filAmount := uint64(math.Ceil(math.Max(math.Log(numSectorInfo)*math.Sqrt(numSectorInfo)/4, 20))) - attoFilAmount := big.Mul(big.NewIntUnsigned(filAmount), big.NewIntUnsigned(build.FilecoinPrecision)) + refundValue := big.Sub(balanceCutoff, totalAvailableBalance) + if r.minerRecoveryRefundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.minerRecoveryRefundPercent))) + } - if totalAvailableBalance.GreaterThanEqual(attoFilAmount) { - log.Debugw("skipping over miner with total available balance larger than refund", "height", tipset.Height(), "key", tipset.Key(), "miner", maddr, "available_balance", totalAvailableBalance, "possible_refund", attoFilAmount) + refundValue = big.Add(refundValue, r.minerRecoveryBonus) + + if refundValue.GreaterThan(r.minerRecoveryCutoff) { + log.Infow( + "skipping over miner with refund greater than refund cutoff", + "height", tipset.Height(), + "key", tipset.Key(), + "miner", maddr, + "available_balance", totalAvailableBalance, + "balance_cutoff", balanceCutoff, + "faults_count", faultsCount, + "refund", refundValue, + "available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "balance_cutoff_fil", big.Div(balanceCutoff, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + ) continue } - refundValue := big.Sub(attoFilAmount, totalAvailableBalance) - refunds.Track(maddr, refundValue) - record := []string{maddr.String(), fmt.Sprintf("%d", len(sectorInfo)), totalAvailableBalance.String(), refundValue.String()} + record := []string{ + maddr.String(), + fmt.Sprintf("%d", faultsCount), + big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).String(), + big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).String(), + } if err := csvOut.Write(record); err != nil { return nil, err } - log.Debugw("processing miner", "miner", maddr, "sectors", len(sectorInfo), "available_balance", totalAvailableBalance, "refund", refundValue) + log.Debugw( + "processing miner", + "miner", maddr, + "faults_count", faultsCount, + "available_balance", totalAvailableBalance, + "refund", refundValue, + "available_balance_fil", big.Div(totalAvailableBalance, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + ) } return refunds, nil @@ -794,17 +916,36 @@ func (r *refunder) ProcessTipset(ctx context.Context, tipset *types.TipSet, refu continue } - if r.percentExtra > 0 { - refundValue = types.BigAdd(refundValue, types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.percentExtra)))) + if r.refundPercent > 0 { + refundValue = types.BigMul(types.BigDiv(refundValue, types.NewInt(100)), types.NewInt(uint64(r.refundPercent))) } - log.Debugw("processing message", "method", messageMethod, "cid", msg.Cid, "from", m.From, "to", m.To, "value", m.Value, "gas_fee_cap", m.GasFeeCap, "gas_premium", m.GasPremium, "gas_used", recps[i].GasUsed, "refund", refundValue) + log.Debugw( + "processing message", + "method", messageMethod, + "cid", msg.Cid, + "from", m.From, + "to", m.To, + "value", m.Value, + "gas_fee_cap", m.GasFeeCap, + "gas_premium", m.GasPremium, + "gas_used", recps[i].GasUsed, + "refund", refundValue, + "refund_fil", big.Div(refundValue, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + ) refunds.Track(m.From, refundValue) tipsetRefunds.Track(m.From, refundValue) } - log.Infow("tipset stats", "height", tipset.Height(), "key", tipset.Key(), "total_refunds", tipsetRefunds.TotalRefunds(), "messages_processed", tipsetRefunds.Count()) + log.Infow( + "tipset stats", + "height", tipset.Height(), + "key", tipset.Key(), + "total_refunds", tipsetRefunds.TotalRefunds(), + "total_refunds_fil", big.Div(tipsetRefunds.TotalRefunds(), big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "messages_processed", tipsetRefunds.Count(), + ) return refunds, nil } @@ -867,13 +1008,24 @@ func (r *refunder) Refund(ctx context.Context, name string, tipset *types.TipSet refundSum = types.BigAdd(refundSum, msg.Value) } - log.Infow(name, "tipsets_processed", rounds, "height", tipset.Height(), "key", tipset.Key(), "messages_sent", len(messages)-failures, "refund_sum", refundSum, "messages_failures", failures, "messages_processed", refunds.Count()) + log.Infow( + name, + "tipsets_processed", rounds, + "height", tipset.Height(), + "key", tipset.Key(), + "messages_sent", len(messages)-failures, + "refund_sum", refundSum, + "refund_sum_fil", big.Div(refundSum, big.NewIntUnsigned(build.FilecoinPrecision)).Int64(), + "messages_failures", failures, + "messages_processed", refunds.Count(), + ) return nil } type Repo struct { - last abi.ChainEpoch - path string + lastHeight abi.ChainEpoch + lastMinerRecoveryHeight abi.ChainEpoch + path string } func NewRepo(path string) (*Repo, error) { @@ -883,8 +1035,9 @@ func NewRepo(path string) (*Repo, error) { } return &Repo{ - last: 0, - path: path, + lastHeight: 0, + lastMinerRecoveryHeight: 0, + path: path, }, nil } @@ -915,43 +1068,66 @@ func (r *Repo) init() error { return nil } -func (r *Repo) Open() (err error) { - if err = r.init(); err != nil { - return +func (r *Repo) Open() error { + if err := r.init(); err != nil { + return err } - var f *os.File + if err := r.loadHeight(); err != nil { + return err + } + + if err := r.loadMinerRecoveryHeight(); err != nil { + return err + } - f, err = os.OpenFile(filepath.Join(r.path, "height"), os.O_RDWR|os.O_CREATE, 0644) + return nil +} + +func loadChainEpoch(fn string) (abi.ChainEpoch, error) { + f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0644) if err != nil { - return + return 0, err } defer func() { err = f.Close() }() - var raw []byte - - raw, err = ioutil.ReadAll(f) + raw, err := ioutil.ReadAll(f) if err != nil { - return + return 0, err } height, err := strconv.Atoi(string(bytes.TrimSpace(raw))) if err != nil { - return + return 0, err } - r.last = abi.ChainEpoch(height) - return + return abi.ChainEpoch(height), nil +} + +func (r *Repo) loadHeight() error { + var err error + r.lastHeight, err = loadChainEpoch(filepath.Join(r.path, "height")) + return err +} + +func (r *Repo) loadMinerRecoveryHeight() error { + var err error + r.lastMinerRecoveryHeight, err = loadChainEpoch(filepath.Join(r.path, "miner_recovery_height")) + return err } func (r *Repo) Height() abi.ChainEpoch { - return r.last + return r.lastHeight +} + +func (r *Repo) MinerRecoveryHeight() abi.ChainEpoch { + return r.lastMinerRecoveryHeight } func (r *Repo) SetHeight(last abi.ChainEpoch) (err error) { - r.last = last + r.lastHeight = last var f *os.File f, err = os.OpenFile(filepath.Join(r.path, "height"), os.O_RDWR, 0644) if err != nil { @@ -962,7 +1138,26 @@ func (r *Repo) SetHeight(last abi.ChainEpoch) (err error) { err = f.Close() }() - if _, err = fmt.Fprintf(f, "%d", r.last); err != nil { + if _, err = fmt.Fprintf(f, "%d", r.lastHeight); err != nil { + return + } + + return +} + +func (r *Repo) SetMinerRecoveryHeight(last abi.ChainEpoch) (err error) { + r.lastMinerRecoveryHeight = last + var f *os.File + f, err = os.OpenFile(filepath.Join(r.path, "miner_recovery_height"), os.O_RDWR, 0644) + if err != nil { + return + } + + defer func() { + err = f.Close() + }() + + if _, err = fmt.Fprintf(f, "%d", r.lastMinerRecoveryHeight); err != nil { return } From 4d4bab12eba36261dca9c422a83f2640c7d7af2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Sep 2020 13:33:42 +0200 Subject: [PATCH 07/27] Improve miner sectors list UX --- cli/util.go | 5 +- cmd/lotus-storage-miner/sectors.go | 111 ++++++++++++++++++++++++----- go.mod | 1 + go.sum | 2 + lib/tablewriter/tablewriter.go | 15 +++- 5 files changed, 111 insertions(+), 23 deletions(-) diff --git a/cli/util.go b/cli/util.go index 762cdb565e7..fb555e320f0 100644 --- a/cli/util.go +++ b/cli/util.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/hako/durafmt" "github.com/ipfs/go-cid" "github.com/filecoin-project/go-state-types/abi" @@ -36,11 +37,11 @@ func parseTipSet(ctx context.Context, api api.FullNode, vals []string) (*types.T func EpochTime(curr, e abi.ChainEpoch) string { switch { case curr > e: - return fmt.Sprintf("%d (%s ago)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e))) + return fmt.Sprintf("%d (%s ago)", e, durafmt.Parse(time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(curr-e))).LimitFirstN(2)) case curr == e: return fmt.Sprintf("%d (now)", e) case curr < e: - return fmt.Sprintf("%d (in %s)", e, time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr))) + return fmt.Sprintf("%d (in %s)", e, durafmt.Parse(time.Second*time.Duration(int64(build.BlockDelaySecs)*int64(e-curr))).LimitFirstN(2)) } panic("math broke") diff --git a/cmd/lotus-storage-miner/sectors.go b/cmd/lotus-storage-miner/sectors.go index 370962bdcd8..b50f4a86d42 100644 --- a/cmd/lotus-storage-miner/sectors.go +++ b/cmd/lotus-storage-miner/sectors.go @@ -5,19 +5,22 @@ import ( "os" "sort" "strconv" - "text/tabwriter" "time" + "github.com/docker/go-units" + "github.com/fatih/color" "github.com/urfave/cli/v2" "golang.org/x/xerrors" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/lib/tablewriter" lcli "github.com/filecoin-project/lotus/cli" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" @@ -144,8 +147,19 @@ var sectorsListCmd = &cli.Command{ Name: "show-removed", Usage: "show removed sectors", }, + &cli.BoolFlag{ + Name: "color", + Aliases: []string{"c"}, + Value: true, + }, + &cli.BoolFlag{ + Name: "fast", + Usage: "don't show on-chain info for better performance", + }, }, Action: func(cctx *cli.Context) error { + color.NoColor = !cctx.Bool("color") + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err @@ -170,7 +184,12 @@ var sectorsListCmd = &cli.Command{ return err } - activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, types.EmptyTSK) + head, err := fullApi.ChainHead(ctx) + if err != nil { + return err + } + + activeSet, err := fullApi.StateMinerActiveSectors(ctx, maddr, head.Key()) if err != nil { return err } @@ -179,7 +198,7 @@ var sectorsListCmd = &cli.Command{ activeIDs[info.SectorNumber] = struct{}{} } - sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, types.EmptyTSK) + sset, err := fullApi.StateMinerSectors(ctx, maddr, nil, head.Key()) if err != nil { return err } @@ -192,12 +211,26 @@ var sectorsListCmd = &cli.Command{ return list[i] < list[j] }) - w := tabwriter.NewWriter(os.Stdout, 8, 4, 1, ' ', 0) + tw := tablewriter.New( + tablewriter.Col("ID"), + tablewriter.Col("State"), + tablewriter.Col("OnChain"), + tablewriter.Col("Active"), + tablewriter.Col("Expiration"), + tablewriter.Col("Deals"), + tablewriter.Col("DealWeight"), + tablewriter.NewLineCol("Error"), + tablewriter.NewLineCol("EarlyExpiration")) + + fast := cctx.Bool("fast") for _, s := range list { - st, err := nodeApi.SectorsStatus(ctx, s, false) + st, err := nodeApi.SectorsStatus(ctx, s, !fast) if err != nil { - fmt.Fprintf(w, "%d:\tError: %s\n", s, err) + tw.Write(map[string]interface{}{ + "ID": s, + "Error": err, + }) continue } @@ -205,20 +238,60 @@ var sectorsListCmd = &cli.Command{ _, inSSet := commitedIDs[s] _, inASet := activeIDs[s] - _, _ = fmt.Fprintf(w, "%d: %s\tsSet: %s\tactive: %s\ttktH: %d\tseedH: %d\tdeals: %v\t toUpgrade:%t\n", - s, - st.State, - yesno(inSSet), - yesno(inASet), - st.Ticket.Epoch, - st.Seed.Epoch, - st.Deals, - st.ToUpgrade, - ) + dw := .0 + if st.Expiration-st.Activation > 0 { + dw = float64(big.Div(st.DealWeight, big.NewInt(int64(st.Expiration-st.Activation))).Uint64()) + } + + var deals int + for _, deal := range st.Deals { + if deal != 0 { + deals++ + } + } + + exp := st.Expiration + if st.OnTime > 0 && st.OnTime < exp { + exp = st.OnTime // Can be different when the sector was CC upgraded + } + + m := map[string]interface{}{ + "ID": s, + "State": color.New(stateOrder[sealing.SectorState(st.State)].col).Sprint(st.State), + "OnChain": yesno(inSSet), + "Active": yesno(inASet), + } + + if deals > 0 { + m["Deals"] = color.GreenString("%d", deals) + } else { + m["Deals"] = color.BlueString("CC") + if st.ToUpgrade { + m["Deals"] = color.CyanString("CC(upgrade)") + } + } + + if !fast { + if !inSSet { + m["Expiration"] = "n/a" + } else { + m["Expiration"] = lcli.EpochTime(head.Height(), exp) + + if !fast && deals > 0 { + m["DealWeight"] = units.BytesSize(dw) + } + + if st.Early > 0 { + m["EarlyExpiration"] = color.YellowString(lcli.EpochTime(head.Height(), st.Early)) + } + } + } + + tw.Write(m) } } - return w.Flush() + return tw.Flush(os.Stdout) }, } @@ -447,7 +520,7 @@ var sectorsUpdateCmd = &cli.Command{ func yesno(b bool) string { if b { - return "YES" + return color.GreenString("YES") } - return "NO" + return color.RedString("NO") } diff --git a/go.mod b/go.mod index b8896d78fb9..b9b133f5127 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.2 + github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/golang-lru v0.5.4 diff --git a/go.sum b/go.sum index 0f309ce1af3..2d9929705d3 100644 --- a/go.sum +++ b/go.sum @@ -415,6 +415,8 @@ github.com/gxed/go-shellwords v1.0.3/go.mod h1:N7paucT91ByIjmVJHhvoarjoQnmsi3Jd3 github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE= +github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 h1:BpJ2o0OR5FV7vrkDYfXYVJQeMNWa8RhklZOpW2ITAIQ= +github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 h1:F9k+7wv5OIk1zcq23QpdiL0hfDuXPjuOmMNaC6fgQ0Q= github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1/go.mod h1:jvfsLIxk0fY/2BKSQ1xf2406AKA5dwMmKKv0ADcOfN8= github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e h1:3YKHER4nmd7b5qy5t0GWDTwSn4OyRgfAXSmo6VnryBY= diff --git a/lib/tablewriter/tablewriter.go b/lib/tablewriter/tablewriter.go index d776113909f..cd045710e39 100644 --- a/lib/tablewriter/tablewriter.go +++ b/lib/tablewriter/tablewriter.go @@ -12,6 +12,7 @@ import ( type Column struct { Name string SeparateLine bool + Lines int } type TableWriter struct { @@ -50,6 +51,7 @@ cloop: for i, column := range w.cols { if column.Name == col { byColID[i] = fmt.Sprint(val) + w.cols[i].Lines++ continue cloop } } @@ -58,6 +60,7 @@ cloop: w.cols = append(w.cols, Column{ Name: col, SeparateLine: false, + Lines: 1, }) } @@ -77,7 +80,11 @@ func (w *TableWriter) Flush(out io.Writer) error { w.rows = append([]map[int]string{header}, w.rows...) - for col := range w.cols { + for col, c := range w.cols { + if c.Lines == 0 { + continue + } + for _, row := range w.rows { val, found := row[col] if !found { @@ -94,9 +101,13 @@ func (w *TableWriter) Flush(out io.Writer) error { cols := make([]string, len(w.cols)) for ci, col := range w.cols { + if col.Lines == 0 { + continue + } + e, _ := row[ci] pad := colLengths[ci] - cliStringLength(e) + 2 - if !col.SeparateLine { + if !col.SeparateLine && col.Lines > 0 { e = e + strings.Repeat(" ", pad) if _, err := fmt.Fprint(out, e); err != nil { return err From d2f279c3de755b267562cb0fdf8bb6b471ad0356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 30 Sep 2020 13:51:52 +0200 Subject: [PATCH 08/27] Print heman-readable transferred bytes in client list-transfers --- cli/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/client.go b/cli/client.go index 5d40dce0d8a..272ab0bdd17 100644 --- a/cli/client.go +++ b/cli/client.go @@ -1423,7 +1423,7 @@ func toChannelOutput(useColor bool, otherPartyColumn string, channel lapi.DataTr otherPartyColumn: otherParty, "Root Cid": rootCid, "Initiated?": initiated, - "Transferred": channel.Transferred, + "Transferred": units.BytesSize(float64(channel.Transferred)), "Voucher": voucher, "Message": channel.Message, } From 5e08d56630adae797e2f94e9279bb77ab3a7f820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 00:27:54 +0200 Subject: [PATCH 09/27] sched: Allow some single-thread tasks to run in parallel with PC2/C2 --- extern/sector-storage/resources.go | 116 ++++++++++++++--------- extern/sector-storage/sched_resources.go | 27 +----- 2 files changed, 75 insertions(+), 68 deletions(-) diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go index 28ab47e6f71..320fec71f0b 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/resources.go @@ -10,14 +10,38 @@ type Resources struct { MinMemory uint64 // What Must be in RAM for decent perf MaxMemory uint64 // Memory required (swap + ram) - Threads int // -1 = multithread - CanGPU bool + MaxParallelism int // -1 = multithread + CanGPU bool BaseMinMemory uint64 // What Must be in RAM for decent perf (shared between threads) } -func (r Resources) MultiThread() bool { - return r.Threads == -1 +/* + + Percent of threads to allocate to parallel tasks + + 12 * 0.92 = 11 + 16 * 0.92 = 14 + 24 * 0.92 = 22 + 32 * 0.92 = 29 + 64 * 0.92 = 58 + 128 * 0.92 = 117 + +*/ +var ParallelNum uint64 = 92 +var ParallelDenom uint64 = 100 + +// TODO: Take NUMA into account +func (r Resources) Threads(wcpus uint64) uint64 { + if r.MaxParallelism == -1 { + n := (wcpus * ParallelNum) / ParallelDenom + if n == 0 { + return wcpus + } + return n + } + + return uint64(r.MaxParallelism) } var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources{ @@ -26,7 +50,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 30, MinMemory: 8 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 1 << 30, }, @@ -34,7 +58,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 4 << 30, MinMemory: 4 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 1 << 30, }, @@ -42,7 +66,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 1 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 1 << 30, }, @@ -50,7 +74,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 2 << 10, }, @@ -58,7 +82,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 8 << 20, }, @@ -68,7 +92,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 128 << 30, MinMemory: 112 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 10 << 20, }, @@ -76,7 +100,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 64 << 30, MinMemory: 56 << 30, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 10 << 20, }, @@ -84,7 +108,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 768 << 20, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 1 << 20, }, @@ -92,7 +116,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 2 << 10, }, @@ -100,7 +124,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: 1, + MaxParallelism: 1, BaseMinMemory: 8 << 20, }, @@ -110,8 +134,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 64 << 30, MinMemory: 64 << 30, - Threads: -1, - CanGPU: true, + MaxParallelism: -1, + CanGPU: true, BaseMinMemory: 60 << 30, }, @@ -119,8 +143,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 32 << 30, MinMemory: 32 << 30, - Threads: -1, - CanGPU: true, + MaxParallelism: -1, + CanGPU: true, BaseMinMemory: 30 << 30, }, @@ -128,7 +152,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 3 << 29, // 1.5G MinMemory: 1 << 30, - Threads: -1, + MaxParallelism: -1, BaseMinMemory: 1 << 30, }, @@ -136,7 +160,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: -1, + MaxParallelism: -1, BaseMinMemory: 2 << 10, }, @@ -144,7 +168,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: -1, + MaxParallelism: -1, BaseMinMemory: 8 << 20, }, @@ -154,7 +178,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 1 << 30, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 1 << 30, }, @@ -162,7 +186,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 1 << 30, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 1 << 30, }, @@ -170,7 +194,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 30, MinMemory: 1 << 30, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 1 << 30, }, @@ -178,7 +202,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 2 << 10, }, @@ -186,7 +210,7 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: 0, + MaxParallelism: 0, BaseMinMemory: 8 << 20, }, @@ -196,8 +220,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 190 << 30, // TODO: Confirm MinMemory: 60 << 30, - Threads: -1, - CanGPU: true, + MaxParallelism: -1, + CanGPU: true, BaseMinMemory: 64 << 30, // params }, @@ -205,8 +229,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 150 << 30, // TODO: ~30G of this should really be BaseMaxMemory MinMemory: 30 << 30, - Threads: -1, - CanGPU: true, + MaxParallelism: -1, + CanGPU: true, BaseMinMemory: 32 << 30, // params }, @@ -214,8 +238,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 3 << 29, // 1.5G MinMemory: 1 << 30, - Threads: 1, // This is fine - CanGPU: true, + MaxParallelism: 1, // This is fine + CanGPU: true, BaseMinMemory: 10 << 30, }, @@ -223,8 +247,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 2 << 10, MinMemory: 2 << 10, - Threads: 1, - CanGPU: true, + MaxParallelism: 1, + CanGPU: true, BaseMinMemory: 2 << 10, }, @@ -232,8 +256,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 8 << 20, MinMemory: 8 << 20, - Threads: 1, - CanGPU: true, + MaxParallelism: 1, + CanGPU: true, BaseMinMemory: 8 << 20, }, @@ -243,8 +267,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, @@ -252,8 +276,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, @@ -261,8 +285,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, @@ -270,8 +294,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, @@ -279,8 +303,8 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources MaxMemory: 1 << 20, MinMemory: 1 << 20, - Threads: 0, - CanGPU: false, + MaxParallelism: 0, + CanGPU: false, BaseMinMemory: 0, }, diff --git a/extern/sector-storage/sched_resources.go b/extern/sector-storage/sched_resources.go index 623472a2055..d6dae577bb7 100644 --- a/extern/sector-storage/sched_resources.go +++ b/extern/sector-storage/sched_resources.go @@ -28,12 +28,7 @@ func (a *activeResources) withResources(id WorkerID, wr storiface.WorkerResource func (a *activeResources) add(wr storiface.WorkerResources, r Resources) { a.gpuUsed = r.CanGPU - if r.MultiThread() { - a.cpuUse += wr.CPUs - } else { - a.cpuUse += uint64(r.Threads) - } - + a.cpuUse += r.Threads(wr.CPUs) a.memUsedMin += r.MinMemory a.memUsedMax += r.MaxMemory } @@ -42,12 +37,7 @@ func (a *activeResources) free(wr storiface.WorkerResources, r Resources) { if r.CanGPU { a.gpuUsed = false } - if r.MultiThread() { - a.cpuUse -= wr.CPUs - } else { - a.cpuUse -= uint64(r.Threads) - } - + a.cpuUse -= r.Threads(wr.CPUs) a.memUsedMin -= r.MinMemory a.memUsedMax -= r.MaxMemory } @@ -68,16 +58,9 @@ func (a *activeResources) canHandleRequest(needRes Resources, wid WorkerID, call return false } - if needRes.MultiThread() { - if a.cpuUse > 0 { - log.Debugf("sched: not scheduling on worker %d for %s; multicore process needs %d threads, %d in use, target %d", wid, caller, res.CPUs, a.cpuUse, res.CPUs) - return false - } - } else { - if a.cpuUse+uint64(needRes.Threads) > res.CPUs { - log.Debugf("sched: not scheduling on worker %d for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads, a.cpuUse, res.CPUs) - return false - } + if a.cpuUse+needRes.Threads(res.CPUs) > res.CPUs { + log.Debugf("sched: not scheduling on worker %d for %s; not enough threads, need %d, %d in use, target %d", wid, caller, needRes.Threads(res.CPUs), a.cpuUse, res.CPUs) + return false } if len(res.GPUs) > 0 && needRes.CanGPU { From 1b7cdb9341c4cf6dce0c23790bb5b2beb68493dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 00:54:34 +0200 Subject: [PATCH 10/27] Fix storage manager tests --- extern/sector-storage/sched_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extern/sector-storage/sched_test.go b/extern/sector-storage/sched_test.go index c560a58f600..579a6d91374 100644 --- a/extern/sector-storage/sched_test.go +++ b/extern/sector-storage/sched_test.go @@ -290,6 +290,9 @@ func TestSched(t *testing.T) { } testFunc := func(workers []workerSpec, tasks []task) func(t *testing.T) { + ParallelNum = 1 + ParallelDenom = 1 + return func(t *testing.T) { index := stores.NewIndex() From 6981f776f4b10fa7c255f86e2467e95d99f6aec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 00:54:53 +0200 Subject: [PATCH 11/27] Lower PC2 memory requirements --- extern/sector-storage/resources.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extern/sector-storage/resources.go b/extern/sector-storage/resources.go index 320fec71f0b..6b531e82b1a 100644 --- a/extern/sector-storage/resources.go +++ b/extern/sector-storage/resources.go @@ -131,22 +131,22 @@ var ResourceTable = map[sealtasks.TaskType]map[abi.RegisteredSealProof]Resources }, sealtasks.TTPreCommit2: { abi.RegisteredSealProof_StackedDrg64GiBV1: Resources{ - MaxMemory: 64 << 30, - MinMemory: 64 << 30, + MaxMemory: 30 << 30, + MinMemory: 30 << 30, MaxParallelism: -1, CanGPU: true, - BaseMinMemory: 60 << 30, + BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg32GiBV1: Resources{ - MaxMemory: 32 << 30, - MinMemory: 32 << 30, + MaxMemory: 15 << 30, + MinMemory: 15 << 30, MaxParallelism: -1, CanGPU: true, - BaseMinMemory: 30 << 30, + BaseMinMemory: 1 << 30, }, abi.RegisteredSealProof_StackedDrg512MiBV1: Resources{ MaxMemory: 3 << 29, // 1.5G From 636810daa5e63a6ec132d78993d028a41f179276 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 21:30:52 -0400 Subject: [PATCH 12/27] Lotus version 0.8.1 --- CHANGELOG.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++++ build/version.go | 2 +- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac687675e7a..6de6ddc2ca2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,71 @@ # Lotus changelog +# 0.8.1 / 2020-09-30 + +This optional release of Lotus introduces a new version of markets which switches to CBOR-map encodings, and allows datastore migrations. The release also introduces several improvements to the mining process, a few performance optimizations, and a battery of UX additions and enhancements. + +## Changes + +#### Dependencies + +- Markets 0.7.0 with updated data stores (https://github.com/filecoin-project/lotus/pull/4089) +- Update ffi to code with blst fixes (https://github.com/filecoin-project/lotus/pull/3998) + +#### Core Lotus + +- Fix GetPower with no miner address (https://github.com/filecoin-project/lotus/pull/4049) +- Refactor: Move nonce generation out of mpool (https://github.com/filecoin-project/lotus/pull/3970) + +#### Performance + +- Implement caching syscalls for import-bench (https://github.com/filecoin-project/lotus/pull/3888) +- Fetch tipset blocks in parallel (https://github.com/filecoin-project/lotus/pull/4074) +- Optimize Tipset equals() (https://github.com/filecoin-project/lotus/pull/4056) +- Make state transition in validation async (https://github.com/filecoin-project/lotus/pull/3868) + +#### Mining + +- Add trace window post (https://github.com/filecoin-project/lotus/pull/4020) +- Use abstract types for Dont recompute post on revert (https://github.com/filecoin-project/lotus/pull/4022) +- Fix injectNulls logic in test miner (https://github.com/filecoin-project/lotus/pull/4058) +- Fix potential panic in FinalizeSector (https://github.com/filecoin-project/lotus/pull/4092) +- Don't recompute post on revert (https://github.com/filecoin-project/lotus/pull/3924) +- Fix some failed precommit handling (https://github.com/filecoin-project/lotus/pull/3445) +- Add --no-swap flag for worker (https://github.com/filecoin-project/lotus/pull/4107) +- Allow some single-thread tasks to run in parallel with PC2/C2 (https://github.com/filecoin-project/lotus/pull/4116) + +#### UX + +- Add an envvar to set address network version (https://github.com/filecoin-project/lotus/pull/4028) +- Add logging to chain export (https://github.com/filecoin-project/lotus/pull/4030) +- Add JSON output to state compute (https://github.com/filecoin-project/lotus/pull/4038) +- Wallet list CLI: Print balances/nonces (https://github.com/filecoin-project/lotus/pull/4088) +- Added an option to show or not show sector info for `lotus-miner info` (https://github.com/filecoin-project/lotus/pull/4003) +- Add a command to import an ipld object into the chainstore (https://github.com/filecoin-project/lotus/pull/3434) +- Improve the lotus-shed dealtracker (https://github.com/filecoin-project/lotus/pull/4051) +- Docs review and re-organization (https://github.com/filecoin-project/lotus/pull/3431) +- Fix wallet list (https://github.com/filecoin-project/lotus/pull/4104) +- Add an endpoint to validate whether a string is a well-formed address (https://github.com/filecoin-project/lotus/pull/4106) +- Add an option to set config path (https://github.com/filecoin-project/lotus/pull/4103) +- Add printf in TestWindowPost (https://github.com/filecoin-project/lotus/pull/4043) +- Improve miner sectors list UX (https://github.com/filecoin-project/lotus/pull/4108) + +#### Tooling + +- Move policy change to seal bench (https://github.com/filecoin-project/lotus/pull/4032) +- Add back network power to stats (https://github.com/filecoin-project/lotus/pull/4050) +- Conformance: Record and feed circulating supply (https://github.com/filecoin-project/lotus/pull/4078) +- Snapshot import progress bar, add HTTP support (https://github.com/filecoin-project/lotus/pull/4070) +- Add lotus shed util to validate a tipset (https://github.com/filecoin-project/lotus/pull/4065) +- tvx: a test vector extraction and execution tool (https://github.com/filecoin-project/lotus/pull/4064) + +#### Bootstrap + +- Add new bootstrappers (https://github.com/filecoin-project/lotus/pull/4007) +- Add Glif node to bootstrap peers (https://github.com/filecoin-project/lotus/pull/4004) +- Add one more node located in China (https://github.com/filecoin-project/lotus/pull/4041) +- Add ipfsmain bootstrapper (https://github.com/filecoin-project/lotus/pull/4067) + # 0.8.0 / 2020-09-26 This consensus-breaking release of Lotus introduces an upgrade to the network. The changes that break consensus are: diff --git a/build/version.go b/build/version.go index 77b98f0084f..0b317aa17f3 100644 --- a/build/version.go +++ b/build/version.go @@ -29,7 +29,7 @@ func buildType() string { } // BuildVersion is the local build version, set by build system -const BuildVersion = "0.8.0" +const BuildVersion = "0.8.1" func UserVersion() string { return BuildVersion + buildType() + CurrentCommit From 60d442e36a6cc6b32da3fa7541537cfee3b7b050 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Wed, 30 Sep 2020 22:05:16 -0400 Subject: [PATCH 13/27] Slash filter shouldn't be trigerred if the same block is submitted multiple times --- chain/gen/slashfilter/slashfilter.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chain/gen/slashfilter/slashfilter.go b/chain/gen/slashfilter/slashfilter.go index fadd3dd27ec..ee04351566d 100644 --- a/chain/gen/slashfilter/slashfilter.go +++ b/chain/gen/slashfilter/slashfilter.go @@ -105,6 +105,10 @@ func checkFault(t ds.Datastore, key ds.Key, bh *types.BlockHeader, faultType str return err } + if other == bh.Cid() { + return nil + } + return xerrors.Errorf("produced block would trigger '%s' consensus fault; miner: %s; bh: %s, other: %s", faultType, bh.Miner, bh.Cid(), other) } From 93e4eae94cfd5cffed88a0ad10358b57f6a2cf11 Mon Sep 17 00:00:00 2001 From: Aayush Rajasekaran Date: Thu, 1 Oct 2020 03:14:59 -0400 Subject: [PATCH 14/27] Some helpers for verifreg work --- api/api_full.go | 6 +++ api/apistruct/struct.go | 10 +++++ chain/actors/builtin/verifreg/v0.go | 4 ++ chain/actors/builtin/verifreg/verifreg.go | 1 + cmd/lotus-shed/verifreg.go | 39 +++++++++--------- documentation/en/api-methods.md | 49 +++++++++++++++++++++++ node/impl/full/state.go | 47 +++++++++++++++++++++- 7 files changed, 137 insertions(+), 19 deletions(-) diff --git a/api/api_full.go b/api/api_full.go index 767739582e1..b443aef3f26 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -390,10 +390,16 @@ type FullNode interface { // StateCompute is a flexible command that applies the given messages on the given tipset. // The messages are run as though the VM were at the provided height. StateCompute(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*ComputeStateOutput, error) + // StateVerifierStatus returns the data cap for the given address. + // Returns nil if there is no entry in the data cap table for the + // address. + StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) // StateVerifiedClientStatus returns the data cap for the given address. // Returns nil if there is no entry in the data cap table for the // address. StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) + // StateVerifiedClientStatus returns the address of the Verified Registry's root key + StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) // StateDealProviderCollateralBounds returns the min and max collateral a storage provider // can issue. It takes the deal size and verified status as parameters. StateDealProviderCollateralBounds(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (DealCollateralBounds, error) diff --git a/api/apistruct/struct.go b/api/apistruct/struct.go index c79d0841c30..7bb0ac86618 100644 --- a/api/apistruct/struct.go +++ b/api/apistruct/struct.go @@ -202,7 +202,9 @@ type FullNodeStruct struct { StateMinerSectorCount func(context.Context, address.Address, types.TipSetKey) (api.MinerSectors, error) `perm:"read"` StateListMessages func(ctx context.Context, match *types.Message, tsk types.TipSetKey, toht abi.ChainEpoch) ([]cid.Cid, error) `perm:"read"` StateCompute func(context.Context, abi.ChainEpoch, []*types.Message, types.TipSetKey) (*api.ComputeStateOutput, error) `perm:"read"` + StateVerifierStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` StateVerifiedClientStatus func(context.Context, address.Address, types.TipSetKey) (*abi.StoragePower, error) `perm:"read"` + StateVerifiedRegistryRootKey func(ctx context.Context, tsk types.TipSetKey) (address.Address, error) `perm:"read"` StateDealProviderCollateralBounds func(context.Context, abi.PaddedPieceSize, bool, types.TipSetKey) (api.DealCollateralBounds, error) `perm:"read"` StateCirculatingSupply func(context.Context, types.TipSetKey) (api.CirculatingSupply, error) `perm:"read"` StateNetworkVersion func(context.Context, types.TipSetKey) (stnetwork.Version, error) `perm:"read"` @@ -893,10 +895,18 @@ func (c *FullNodeStruct) StateCompute(ctx context.Context, height abi.ChainEpoch return c.Internal.StateCompute(ctx, height, msgs, tsk) } +func (c *FullNodeStruct) StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { + return c.Internal.StateVerifierStatus(ctx, addr, tsk) +} + func (c *FullNodeStruct) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { return c.Internal.StateVerifiedClientStatus(ctx, addr, tsk) } +func (c *FullNodeStruct) StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) { + return c.Internal.StateVerifiedRegistryRootKey(ctx, tsk) +} + func (c *FullNodeStruct) StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (api.DealCollateralBounds, error) { return c.Internal.StateDealProviderCollateralBounds(ctx, size, verified, tsk) } diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index c59a588114b..51ed3b45679 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -39,6 +39,10 @@ func getDataCap(store adt.Store, root cid.Cid, addr address.Address) (bool, abi. return true, dcap, nil } +func (s *state0) RootKey() (address.Address, error) { + return s.State.RootKey, nil +} + func (s *state0) VerifiedClientDataCap(addr address.Address) (bool, abi.StoragePower, error) { return getDataCap(s.store, s.State.VerifiedClients, addr) } diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index c861f862f21..95a60d1d99b 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -30,6 +30,7 @@ func Load(store adt.Store, act *types.Actor) (State, error) { type State interface { cbor.Marshaler + RootKey() (address.Address, error) VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) VerifierDataCap(address.Address) (bool, abi.StoragePower, error) ForEachVerifier(func(addr address.Address, dcap abi.StoragePower) error) error diff --git a/cmd/lotus-shed/verifreg.go b/cmd/lotus-shed/verifreg.go index 3e2f34f4b14..860498302f9 100644 --- a/cmd/lotus-shed/verifreg.go +++ b/cmd/lotus-shed/verifreg.go @@ -3,6 +3,8 @@ package main import ( "fmt" + "github.com/filecoin-project/go-state-types/big" + "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -37,29 +39,31 @@ var verifRegCmd = &cli.Command{ } var verifRegAddVerifierCmd = &cli.Command{ - Name: "add-verifier", - Usage: "make a given account a verifier", + Name: "add-verifier", + Usage: "make a given account a verifier", + ArgsUsage: " ", Action: func(cctx *cli.Context) error { - fromk, err := address.NewFromString("t3qfoulel6fy6gn3hjmbhpdpf6fs5aqjb5fkurhtwvgssizq4jey5nw4ptq5up6h7jk7frdvvobv52qzmgjinq") - if err != nil { - return err + if cctx.Args().Len() != 3 { + return fmt.Errorf("must specify three arguments: sender, verifier, and allowance") } - if cctx.Args().Len() != 2 { - return fmt.Errorf("must specify two arguments: address and allowance") + sender, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err } - target, err := address.NewFromString(cctx.Args().Get(0)) + verifier, err := address.NewFromString(cctx.Args().Get(1)) if err != nil { return err } - allowance, err := types.BigFromString(cctx.Args().Get(1)) + allowance, err := types.BigFromString(cctx.Args().Get(2)) if err != nil { return err } - params, err := actors.SerializeParams(&verifreg0.AddVerifierParams{Address: target, Allowance: allowance}) + // TODO: ActorUpgrade: Abstract + params, err := actors.SerializeParams(&verifreg0.AddVerifierParams{Address: verifier, Allowance: allowance}) if err != nil { return err } @@ -71,21 +75,19 @@ var verifRegAddVerifierCmd = &cli.Command{ defer closer() ctx := lcli.ReqContext(cctx) - msg := &types.Message{ - To: verifreg.Address, - From: fromk, - Method: builtin0.MethodsVerifiedRegistry.AddVerifier, - Params: params, + vrk, err := api.StateVerifiedRegistryRootKey(ctx, types.EmptyTSK) + if err != nil { + return err } - smsg, err := api.MpoolPushMessage(ctx, msg, nil) + smsg, err := api.MsigPropose(ctx, vrk, verifreg.Address, big.Zero(), sender, uint64(builtin0.MethodsVerifiedRegistry.AddVerifier), params) if err != nil { return err } - fmt.Printf("message sent, now waiting on cid: %s\n", smsg.Cid()) + fmt.Printf("message sent, now waiting on cid: %s\n", smsg) - mwait, err := api.StateWaitMsg(ctx, smsg.Cid(), build.MessageConfidence) + mwait, err := api.StateWaitMsg(ctx, smsg, build.MessageConfidence) if err != nil { return err } @@ -94,6 +96,7 @@ var verifRegAddVerifierCmd = &cli.Command{ return fmt.Errorf("failed to add verifier: %d", mwait.Receipt.ExitCode) } + //TODO: Internal msg might still have failed return nil }, diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2b28816f7a2..ff01f648eb3 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -160,6 +160,8 @@ * [StateSectorPartition](#StateSectorPartition) * [StateSectorPreCommitInfo](#StateSectorPreCommitInfo) * [StateVerifiedClientStatus](#StateVerifiedClientStatus) + * [StateVerifiedRegistryRootKey](#StateVerifiedRegistryRootKey) + * [StateVerifierStatus](#StateVerifierStatus) * [StateWaitMsg](#StateWaitMsg) * [Sync](#Sync) * [SyncCheckBad](#SyncCheckBad) @@ -4117,6 +4119,53 @@ Returns nil if there is no entry in the data cap table for the address. +Perms: read + +Inputs: +```json +[ + "t01234", + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"0"` + +### StateVerifiedRegistryRootKey +StateVerifiedClientStatus returns the address of the Verified Registry's root key + + +Perms: read + +Inputs: +```json +[ + [ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + }, + { + "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" + } + ] +] +``` + +Response: `"t01234"` + +### StateVerifierStatus +StateVerifierStatus returns the data cap for the given address. +Returns nil if there is no entry in the data cap table for the +address. + + Perms: read Inputs: diff --git a/node/impl/full/state.go b/node/impl/full/state.go index f8bf92a926f..e5bd9f9d88c 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -1021,11 +1021,42 @@ func (a *StateAPI) StateMinerAvailableBalance(ctx context.Context, maddr address return types.BigAdd(abal, vested), nil } +// StateVerifiedClientStatus returns the data cap for the given address. +// Returns zero if there is no entry in the data cap table for the +// address. +func (a *StateAPI) StateVerifierStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { + act, err := a.StateGetActor(ctx, verifreg.Address, tsk) + if err != nil { + return nil, err + } + + aid, err := a.StateLookupID(ctx, addr, tsk) + if err != nil { + log.Warnf("lookup failure %v", err) + return nil, err + } + + vrs, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), act) + if err != nil { + return nil, xerrors.Errorf("failed to load verified registry state: %w", err) + } + + verified, dcap, err := vrs.VerifierDataCap(aid) + if err != nil { + return nil, xerrors.Errorf("looking up verifier: %w", err) + } + if !verified { + return nil, nil + } + + return &dcap, nil +} + // StateVerifiedClientStatus returns the data cap for the given address. // Returns zero if there is no entry in the data cap table for the // address. func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*abi.StoragePower, error) { - act, err := a.StateGetActor(ctx, builtin0.VerifiedRegistryActorAddr, tsk) + act, err := a.StateGetActor(ctx, verifreg.Address, tsk) if err != nil { return nil, err } @@ -1052,6 +1083,20 @@ func (a *StateAPI) StateVerifiedClientStatus(ctx context.Context, addr address.A return &dcap, nil } +func (a *StateAPI) StateVerifiedRegistryRootKey(ctx context.Context, tsk types.TipSetKey) (address.Address, error) { + vact, err := a.StateGetActor(ctx, verifreg.Address, tsk) + if err != nil { + return address.Undef, err + } + + vst, err := verifreg.Load(a.StateManager.ChainStore().Store(ctx), vact) + if err != nil { + return address.Undef, err + } + + return vst.RootKey() +} + var dealProviderCollateralNum = types.NewInt(110) var dealProviderCollateralDen = types.NewInt(100) From 9dc75a7bc46a013bb388eec9c12896a4050eb5fe Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 14 Sep 2020 10:11:11 +0200 Subject: [PATCH 15/27] fix: verify voucher amount check --- api/test/paych.go | 47 ++++- chain/actors/builtin/paych/mock/mock.go | 3 +- paychmgr/mock_test.go | 2 +- paychmgr/paych.go | 29 ++- paychmgr/paych_test.go | 234 +++++++----------------- paychmgr/paychget_test.go | 2 +- paychmgr/simple.go | 2 +- 7 files changed, 130 insertions(+), 189 deletions(-) diff --git a/api/test/paych.go b/api/test/paych.go index 36eb2c25622..15ce352bd5f 100644 --- a/api/test/paych.go +++ b/api/test/paych.go @@ -67,7 +67,7 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { t.Fatal(err) } - channelAmt := int64(100000) + channelAmt := int64(7000) channelInfo, err := paymentCreator.PaychGet(ctx, createrAddr, receiverAddr, abi.NewTokenAmount(channelAmt)) if err != nil { t.Fatal(err) @@ -169,6 +169,51 @@ func TestPaymentChannels(t *testing.T, b APIBuilder, blocktime time.Duration) { t.Fatal("Timed out waiting for receiver to submit vouchers") } + // Create a new voucher now that some vouchers have already been submitted + vouchRes, err := paymentCreator.PaychVoucherCreate(ctx, channel, abi.NewTokenAmount(1000), 3) + if err != nil { + t.Fatal(err) + } + if vouchRes.Voucher == nil { + t.Fatal(fmt.Errorf("Not enough funds to create voucher: missing %d", vouchRes.Shortfall)) + } + vdelta, err := paymentReceiver.PaychVoucherAdd(ctx, channel, vouchRes.Voucher, nil, abi.NewTokenAmount(1000)) + if err != nil { + t.Fatal(err) + } + if !vdelta.Equals(abi.NewTokenAmount(1000)) { + t.Fatal("voucher didn't have the right amount") + } + + // Create a new voucher whose value would exceed the channel balance + excessAmt := abi.NewTokenAmount(1000) + vouchRes, err = paymentCreator.PaychVoucherCreate(ctx, channel, excessAmt, 4) + if err != nil { + t.Fatal(err) + } + if vouchRes.Voucher != nil { + t.Fatal("Expected not to be able to create voucher whose value would exceed channel balance") + } + if !vouchRes.Shortfall.Equals(excessAmt) { + t.Fatal(fmt.Errorf("Expected voucher shortfall of %d, got %d", excessAmt, vouchRes.Shortfall)) + } + + // Add a voucher whose value would exceed the channel balance + vouch := &paych.SignedVoucher{ChannelAddr: channel, Amount: excessAmt, Lane: 4, Nonce: 1} + vb, err := vouch.SigningBytes() + if err != nil { + t.Fatal(err) + } + sig, err := paymentCreator.WalletSign(ctx, createrAddr, vb) + if err != nil { + t.Fatal(err) + } + vouch.Signature = sig + _, err = paymentReceiver.PaychVoucherAdd(ctx, channel, vouch, nil, abi.NewTokenAmount(1000)) + if err == nil { + t.Fatal(fmt.Errorf("Expected shortfall error of %d", excessAmt)) + } + // wait for the settlement period to pass before collecting waitForBlocks(ctx, t, bm, paymentReceiver, receiverAddr, paych.SettleDelay) diff --git a/chain/actors/builtin/paych/mock/mock.go b/chain/actors/builtin/paych/mock/mock.go index c4903f3ac25..3b82511ffa0 100644 --- a/chain/actors/builtin/paych/mock/mock.go +++ b/chain/actors/builtin/paych/mock/mock.go @@ -27,10 +27,9 @@ type mockLaneState struct { func NewMockPayChState(from address.Address, to address.Address, settlingAt abi.ChainEpoch, - toSend abi.TokenAmount, lanes map[uint64]paych.LaneState, ) paych.State { - return &mockState{from, to, settlingAt, toSend, lanes} + return &mockState{from: from, to: to, settlingAt: settlingAt, toSend: big.NewInt(0), lanes: lanes} } // NewMockLaneState constructs a state for a payment channel lane with the set fixed values diff --git a/paychmgr/mock_test.go b/paychmgr/mock_test.go index c761221d2ad..3232e08938e 100644 --- a/paychmgr/mock_test.go +++ b/paychmgr/mock_test.go @@ -9,7 +9,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" - "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" @@ -57,6 +56,7 @@ func (sm *mockStateManager) setAccountAddress(a address.Address, lookup address. func (sm *mockStateManager) setPaychState(a address.Address, actor *types.Actor, state paych.State) { sm.lk.Lock() defer sm.lk.Unlock() + sm.paychState[a] = mockPchState{actor, state} } diff --git a/paychmgr/paych.go b/paychmgr/paych.go index f856b98900d..2cf20098677 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -205,7 +205,7 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add } // Check the voucher against the highest known voucher nonce / value - laneStates, err := ca.laneState(ctx, pchState, ch) + laneStates, err := ca.laneState(pchState, ch) if err != nil { return nil, err } @@ -253,16 +253,9 @@ func (ca *channelAccessor) checkVoucherValidUnlocked(ctx context.Context, ch add return nil, err } - // Total required balance = total redeemed + toSend - // Must not exceed actor balance - ts, err := pchState.ToSend() - if err != nil { - return nil, err - } - - newTotal := types.BigAdd(totalRedeemed, ts) - if act.Balance.LessThan(newTotal) { - return nil, newErrInsufficientFunds(types.BigSub(newTotal, act.Balance)) + // Total required balance must not exceed actor balance + if act.Balance.LessThan(totalRedeemed) { + return nil, newErrInsufficientFunds(types.BigSub(totalRedeemed, act.Balance)) } if len(sv.Merges) != 0 { @@ -505,7 +498,6 @@ func (ca *channelAccessor) allocateLane(ch address.Address) (uint64, error) { ca.lk.Lock() defer ca.lk.Unlock() - // TODO: should this take into account lane state? return ca.store.AllocateLane(ch) } @@ -520,7 +512,7 @@ func (ca *channelAccessor) listVouchers(ctx context.Context, ch address.Address) // laneState gets the LaneStates from chain, then applies all vouchers in // the data store over the chain state -func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch address.Address) (map[uint64]paych.LaneState, error) { +func (ca *channelAccessor) laneState(state paych.State, ch address.Address) (map[uint64]paych.LaneState, error) { // TODO: we probably want to call UpdateChannelState with all vouchers to be fully correct // (but technically dont't need to) @@ -547,14 +539,20 @@ func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch return nil, err } + // Note: we use a map instead of an array to store laneStates because the + // client sets the lane ID (the index) and potentially they could use a + // very large index. for _, v := range vouchers { for range v.Voucher.Merges { return nil, xerrors.Errorf("paych merges not handled yet") } - // If there's a voucher for a lane that isn't in chain state just - // create it + // Check if there is an existing laneState in the payment channel + // for this voucher's lane ls, ok := laneStates[v.Voucher.Lane] + + // If the voucher does not have a higher nonce than the existing + // laneState for this lane, ignore it if ok { n, err := ls.Nonce() if err != nil { @@ -565,6 +563,7 @@ func (ca *channelAccessor) laneState(ctx context.Context, state paych.State, ch } } + // Voucher has a higher nonce, so replace laneState with this voucher laneStates[v.Voucher.Lane] = laneState{v.Voucher.Amount, v.Voucher.Nonce} } diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index b27b1e540b1..71c2bd854e6 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -47,7 +47,6 @@ func TestCheckVoucherValid(t *testing.T) { expectError bool key []byte actorBalance big.Int - toSend big.Int voucherAmount big.Int voucherLane uint64 voucherNonce uint64 @@ -56,35 +55,30 @@ func TestCheckVoucherValid(t *testing.T) { name: "passes when voucher amount < balance", key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), }, { name: "fails when funds too low", expectError: true, key: fromKeyPrivate, actorBalance: big.NewInt(5), - toSend: big.NewInt(0), voucherAmount: big.NewInt(10), }, { name: "fails when invalid signature", expectError: true, key: randKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), }, { name: "fails when signed by channel To account (instead of From account)", expectError: true, key: toKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), }, { name: "fails when nonce too low", expectError: true, key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), voucherLane: 1, voucherNonce: 2, @@ -95,7 +89,6 @@ func TestCheckVoucherValid(t *testing.T) { name: "passes when nonce higher", key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), voucherLane: 1, voucherNonce: 3, @@ -106,7 +99,6 @@ func TestCheckVoucherValid(t *testing.T) { name: "passes when nonce for different lane", key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), voucherLane: 2, voucherNonce: 2, @@ -118,32 +110,22 @@ func TestCheckVoucherValid(t *testing.T) { expectError: true, key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(0), voucherAmount: big.NewInt(5), voucherLane: 1, voucherNonce: 3, laneStates: map[uint64]paych.LaneState{ 1: paychmock.NewMockLaneState(big.NewInt(6), 2), }, - }, { - name: "fails when voucher + ToSend > balance", - expectError: true, - key: fromKeyPrivate, - actorBalance: big.NewInt(10), - toSend: big.NewInt(9), - voucherAmount: big.NewInt(2), }, { // voucher supersedes lane 1 redeemed so // lane 1 effective redeemed = voucher amount // - // required balance = toSend + total redeemed - // = 1 + 6 (lane1) + // required balance = voucher amt // = 7 // So required balance: 7 < actor balance: 10 - name: "passes when voucher + total redeemed <= balance", + name: "passes when voucher total redeemed <= balance", key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(1), voucherAmount: big.NewInt(6), voucherLane: 1, voucherNonce: 2, @@ -152,29 +134,68 @@ func TestCheckVoucherValid(t *testing.T) { 1: paychmock.NewMockLaneState(big.NewInt(4), 1), }, }, { - // required balance = toSend + total redeemed - // = 1 + 4 (lane 2) + 6 (voucher lane 1) + // required balance = total redeemed + // = 6 (voucher lane 1) + 5 (lane 2) // = 11 // So required balance: 11 > actor balance: 10 - name: "fails when voucher + total redeemed > balance", + name: "fails when voucher total redeemed > balance", expectError: true, key: fromKeyPrivate, actorBalance: big.NewInt(10), - toSend: big.NewInt(1), voucherAmount: big.NewInt(6), voucherLane: 1, voucherNonce: 1, laneStates: map[uint64]paych.LaneState{ // Lane 2 (different from voucher lane 1) - 2: paychmock.NewMockLaneState(big.NewInt(4), 1), + 2: paychmock.NewMockLaneState(big.NewInt(5), 1), + }, + }, { + // voucher supersedes lane 1 redeemed so + // lane 1 effective redeemed = voucher amount + // + // required balance = total redeemed + // = 6 (new voucher lane 1) + 5 (lane 2) + // = 11 + // So required balance: 11 > actor balance: 10 + name: "fails when voucher total redeemed > balance", + expectError: true, + key: fromKeyPrivate, + actorBalance: big.NewInt(10), + voucherAmount: big.NewInt(6), + voucherLane: 1, + voucherNonce: 2, + laneStates: map[uint64]paych.LaneState{ + // Lane 1 (superseded by new voucher in voucher lane 1) + 1: paychmock.NewMockLaneState(big.NewInt(5), 1), + // Lane 2 (different from voucher lane 1) + 2: paychmock.NewMockLaneState(big.NewInt(5), 1), + }, + }, { + // voucher supersedes lane 1 redeemed so + // lane 1 effective redeemed = voucher amount + // + // required balance = total redeemed + // = 5 (new voucher lane 1) + 5 (lane 2) + // = 10 + // So required balance: 10 <= actor balance: 10 + name: "passes when voucher total redeemed <= balance", + expectError: false, + key: fromKeyPrivate, + actorBalance: big.NewInt(10), + voucherAmount: big.NewInt(5), + voucherLane: 1, + voucherNonce: 2, + laneStates: map[uint64]paych.LaneState{ + // Lane 1 (superseded by new voucher in voucher lane 1) + 1: paychmock.NewMockLaneState(big.NewInt(4), 1), + // Lane 2 (different from voucher lane 1) + 2: paychmock.NewMockLaneState(big.NewInt(5), 1), }, }} for _, tcase := range tcases { tcase := tcase t.Run(tcase.name, func(t *testing.T) { - store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) - // Create an actor for the channel with the test case balance act := &types.Actor{ Code: builtin.AccountActorCodeID, @@ -184,16 +205,17 @@ func TestCheckVoucherValid(t *testing.T) { } mock.setPaychState(ch, act, paychmock.NewMockPayChState( - fromAcct, toAcct, abi.ChainEpoch(0), tcase.toSend, tcase.laneStates)) + fromAcct, toAcct, abi.ChainEpoch(0), tcase.laneStates)) // Create a manager + store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) mgr, err := newManager(store, mock) require.NoError(t, err) // Add channel To address to wallet mock.addWalletAddress(to) - // Create a signed voucher + // Create the test case signed voucher sv := createTestVoucher(t, ch, tcase.voucherLane, tcase.voucherNonce, tcase.voucherAmount, tcase.key) // Check the voucher's validity @@ -207,135 +229,11 @@ func TestCheckVoucherValid(t *testing.T) { } } -func TestCheckVoucherValidCountingAllLanes(t *testing.T) { - ctx := context.Background() - - fromKeyPrivate, fromKeyPublic := testGenerateKeyPair(t) - - ch := tutils.NewIDAddr(t, 100) - from := tutils.NewSECP256K1Addr(t, string(fromKeyPublic)) - to := tutils.NewSECP256K1Addr(t, "secpTo") - fromAcct := tutils.NewActorAddr(t, "fromAct") - toAcct := tutils.NewActorAddr(t, "toAct") - minDelta := big.NewInt(0) - - mock := newMockManagerAPI() - mock.setAccountAddress(fromAcct, from) - mock.setAccountAddress(toAcct, to) - - store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) - - actorBalance := big.NewInt(10) - toSend := big.NewInt(1) - laneStates := map[uint64]paych.LaneState{ - 1: paychmock.NewMockLaneState(big.NewInt(3), 1), - 2: paychmock.NewMockLaneState(big.NewInt(4), 1), - } - - act := &types.Actor{ - Code: builtin.AccountActorCodeID, - Head: cid.Cid{}, - Nonce: 0, - Balance: actorBalance, - } - - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), toSend, laneStates)) - - mgr, err := newManager(store, mock) - require.NoError(t, err) - - // Add channel To address to wallet - mock.addWalletAddress(to) - - // - // Should not be possible to add a voucher with a value such that - // + toSend > - // - // lane 1 redeemed: 3 - // voucher amount (lane 1): 6 - // lane 1 redeemed (with voucher): 6 - // - // Lane 1: 6 - // Lane 2: 4 - // toSend: 1 - // -- - // total: 11 - // - // actor balance is 10 so total is too high. - // - voucherLane := uint64(1) - voucherNonce := uint64(2) - voucherAmount := big.NewInt(6) - sv := createTestVoucher(t, ch, voucherLane, voucherNonce, voucherAmount, fromKeyPrivate) - err = mgr.CheckVoucherValid(ctx, ch, sv) - require.Error(t, err) - - // - // lane 1 redeemed: 3 - // voucher amount (lane 1): 4 - // lane 1 redeemed (with voucher): 4 - // - // Lane 1: 4 - // Lane 2: 4 - // toSend: 1 - // -- - // total: 9 - // - // actor balance is 10 so total is ok. - // - voucherAmount = big.NewInt(4) - sv = createTestVoucher(t, ch, voucherLane, voucherNonce, voucherAmount, fromKeyPrivate) - err = mgr.CheckVoucherValid(ctx, ch, sv) - require.NoError(t, err) - - // Add voucher to lane 1, so Lane 1 effective redeemed - // (with first voucher) is now 4 - _, err = mgr.AddVoucherOutbound(ctx, ch, sv, nil, minDelta) - require.NoError(t, err) - - // - // lane 1 redeemed: 4 - // voucher amount (lane 1): 6 - // lane 1 redeemed (with voucher): 6 - // - // Lane 1: 6 - // Lane 2: 4 - // toSend: 1 - // -- - // total: 11 - // - // actor balance is 10 so total is too high. - // - voucherNonce++ - voucherAmount = big.NewInt(6) - sv = createTestVoucher(t, ch, voucherLane, voucherNonce, voucherAmount, fromKeyPrivate) - err = mgr.CheckVoucherValid(ctx, ch, sv) - require.Error(t, err) - - // - // lane 1 redeemed: 4 - // voucher amount (lane 1): 5 - // lane 1 redeemed (with voucher): 5 - // - // Lane 1: 5 - // Lane 2: 4 - // toSend: 1 - // -- - // total: 10 - // - // actor balance is 10 so total is ok. - // - voucherAmount = big.NewInt(5) - sv = createTestVoucher(t, ch, voucherLane, voucherNonce, voucherAmount, fromKeyPrivate) - err = mgr.CheckVoucherValid(ctx, ch, sv) - require.NoError(t, err) -} - func TestCreateVoucher(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // Create a voucher in lane 1 voucherLane1Amt := big.NewInt(5) @@ -400,7 +298,7 @@ func TestAddVoucherDelta(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) voucherLane := uint64(1) @@ -442,7 +340,7 @@ func TestAddVoucherNextLane(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) minDelta := big.NewInt(0) voucherAmount := big.NewInt(2) @@ -489,10 +387,8 @@ func TestAddVoucherNextLane(t *testing.T) { } func TestAllocateLane(t *testing.T) { - ctx := context.Background() - // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // First lane should be 0 lane, err := s.mgr.AllocateLane(s.ch) @@ -525,7 +421,6 @@ func TestAllocateLaneWithExistingLaneState(t *testing.T) { // Create a channel that will be retrieved from state actorBalance := big.NewInt(10) - toSend := big.NewInt(1) act := &types.Actor{ Code: builtin.AccountActorCodeID, @@ -534,7 +429,7 @@ func TestAllocateLaneWithExistingLaneState(t *testing.T) { Balance: actorBalance, } - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), toSend, make(map[uint64]paych.LaneState))) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) mgr, err := newManager(store, mock) require.NoError(t, err) @@ -559,7 +454,7 @@ func TestAddVoucherProof(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) nonce := uint64(1) voucherAmount := big.NewInt(1) @@ -622,10 +517,11 @@ func TestAddVoucherInboundWalletKey(t *testing.T) { } mock := newMockManagerAPI() + mock.setAccountAddress(fromAcct, from) mock.setAccountAddress(toAcct, to) - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), types.NewInt(0), make(map[uint64]paych.LaneState))) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) // Create a manager store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) @@ -660,7 +556,7 @@ func TestBestSpendable(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // Add vouchers to lane 1 with amounts: [1, 2, 3] voucherLane := uint64(1) @@ -740,7 +636,7 @@ func TestCheckSpendable(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // Create voucher with Extra voucherLane := uint64(1) @@ -821,7 +717,7 @@ func TestSubmitVoucher(t *testing.T) { ctx := context.Background() // Set up a manager with a single payment channel - s := testSetupMgrWithChannel(ctx, t) + s := testSetupMgrWithChannel(t) // Create voucher with Extra voucherLane := uint64(1) @@ -908,7 +804,7 @@ type testScaffold struct { fromKeyPrivate []byte } -func testSetupMgrWithChannel(ctx context.Context, t *testing.T) *testScaffold { +func testSetupMgrWithChannel(t *testing.T) *testScaffold { fromKeyPrivate, fromKeyPublic := testGenerateKeyPair(t) ch := tutils.NewIDAddr(t, 100) @@ -920,6 +816,8 @@ func testSetupMgrWithChannel(ctx context.Context, t *testing.T) *testScaffold { mock := newMockManagerAPI() mock.setAccountAddress(fromAcct, from) mock.setAccountAddress(toAcct, to) + //mock.setAccountState(fromAcct, account.State{Address: from}) + //mock.setAccountState(toAcct, account.State{Address: to}) // Create channel in state balance := big.NewInt(20) @@ -929,7 +827,7 @@ func testSetupMgrWithChannel(ctx context.Context, t *testing.T) *testScaffold { Nonce: 0, Balance: balance, } - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), big.NewInt(0), make(map[uint64]paych.LaneState))) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) mgr, err := newManager(store, mock) diff --git a/paychmgr/paychget_test.go b/paychmgr/paychget_test.go index 430e66c6797..e35ae837110 100644 --- a/paychmgr/paychget_test.go +++ b/paychmgr/paychget_test.go @@ -978,7 +978,7 @@ func TestPaychAvailableFunds(t *testing.T) { Nonce: 0, Balance: createAmt, } - mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), big.NewInt(0), make(map[uint64]paych.LaneState))) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) // Send create channel response response := testChannelResponse(t, ch) mock.receiveMsgResponse(createMsgCid, response) diff --git a/paychmgr/simple.go b/paychmgr/simple.go index d49ccafe605..ca778829fba 100644 --- a/paychmgr/simple.go +++ b/paychmgr/simple.go @@ -312,7 +312,7 @@ func (ca *channelAccessor) currentAvailableFunds(channelID string, queuedAmt typ return nil, err } - laneStates, err := ca.laneState(ca.chctx, pchState, ch) + laneStates, err := ca.laneState(pchState, ch) if err != nil { return nil, err } From e2ecc35dffdc290df95554b5c9ccfd8d02fccd61 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Mon, 14 Sep 2020 11:49:58 +0200 Subject: [PATCH 16/27] test: add test for voucher after add-funds --- paychmgr/paychvoucherfunds_test.go | 107 +++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 paychmgr/paychvoucherfunds_test.go diff --git a/paychmgr/paychvoucherfunds_test.go b/paychmgr/paychvoucherfunds_test.go new file mode 100644 index 00000000000..735ea54aceb --- /dev/null +++ b/paychmgr/paychvoucherfunds_test.go @@ -0,0 +1,107 @@ +package paychmgr + +import ( + "context" + "testing" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/specs-actors/actors/builtin" + "github.com/filecoin-project/specs-actors/actors/builtin/account" + paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" + tutils "github.com/filecoin-project/specs-actors/support/testing" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" + "github.com/stretchr/testify/require" +) + +// TestPaychAddVoucherAfterAddFunds tests adding a voucher to a channel with +// insufficient funds, then adding funds to the channel, then adding the +// voucher again +func TestPaychAddVoucherAfterAddFunds(t *testing.T) { + ctx := context.Background() + store := NewStore(ds_sync.MutexWrap(ds.NewMapDatastore())) + + fromKeyPrivate, fromKeyPublic := testGenerateKeyPair(t) + ch := tutils.NewIDAddr(t, 100) + from := tutils.NewSECP256K1Addr(t, string(fromKeyPublic)) + to := tutils.NewSECP256K1Addr(t, "secpTo") + fromAcct := tutils.NewActorAddr(t, "fromAct") + toAcct := tutils.NewActorAddr(t, "toAct") + + mock := newMockManagerAPI() + defer mock.close() + + // Add the from signing key to the wallet + mock.setAccountState(fromAcct, account.State{Address: from}) + mock.setAccountState(toAcct, account.State{Address: to}) + mock.addSigningKey(fromKeyPrivate) + + mgr, err := newManager(store, mock) + require.NoError(t, err) + + // Send create message for a channel with value 10 + createAmt := big.NewInt(10) + _, createMsgCid, err := mgr.GetPaych(ctx, from, to, createAmt) + require.NoError(t, err) + + // Send create channel response + response := testChannelResponse(t, ch) + mock.receiveMsgResponse(createMsgCid, response) + + // Create an actor in state for the channel with the initial channel balance + act := &types.Actor{ + Code: builtin.AccountActorCodeID, + Head: cid.Cid{}, + Nonce: 0, + Balance: createAmt, + } + mock.setPaychState(ch, act, paych.State{ + From: fromAcct, + To: toAcct, + SettlingAt: abi.ChainEpoch(0), + MinSettleHeight: abi.ChainEpoch(0), + }) + + // Wait for create response to be processed by manager + _, err = mgr.GetPaychWaitReady(ctx, createMsgCid) + require.NoError(t, err) + + // Create a voucher with a value equal to the channel balance + voucher := paych.SignedVoucher{Amount: createAmt, Lane: 1} + res, err := mgr.CreateVoucher(ctx, ch, voucher) + require.NoError(t, err) + require.NotNil(t, res.Voucher) + + // Create a voucher in a different lane with an amount that exceeds the + // channel balance + excessAmt := types.NewInt(5) + voucher = paych.SignedVoucher{Amount: excessAmt, Lane: 2} + res, err = mgr.CreateVoucher(ctx, ch, voucher) + require.NoError(t, err) + require.Nil(t, res.Voucher) + require.Equal(t, res.Shortfall, excessAmt) + + // Add funds so as to cover the voucher shortfall + _, addFundsMsgCid, err := mgr.GetPaych(ctx, from, to, excessAmt) + require.NoError(t, err) + + // Trigger add funds confirmation + mock.receiveMsgResponse(addFundsMsgCid, types.MessageReceipt{ExitCode: 0}) + + // Update actor test case balance to reflect added funds + act.Balance = types.BigAdd(createAmt, excessAmt) + + // Wait for add funds confirmation to be processed by manager + _, err = mgr.GetPaychWaitReady(ctx, addFundsMsgCid) + require.NoError(t, err) + + // Adding same voucher that previously exceeded channel balance + // should succeed now that the channel balance has been increased + res, err = mgr.CreateVoucher(ctx, ch, voucher) + require.NoError(t, err) + require.NotNil(t, res.Voucher) +} From 594ec8855c4cc9a4932a22d4463c2c970b6cc6e1 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Tue, 15 Sep 2020 20:06:33 +0200 Subject: [PATCH 17/27] fix: read lane state from chain as well as datastore --- paychmgr/mock_test.go | 2 +- paychmgr/paych.go | 3 --- paychmgr/paych_test.go | 2 -- paychmgr/paychvoucherfunds_test.go | 16 ++++++---------- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/paychmgr/mock_test.go b/paychmgr/mock_test.go index 3232e08938e..c761221d2ad 100644 --- a/paychmgr/mock_test.go +++ b/paychmgr/mock_test.go @@ -9,6 +9,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/actors/builtin/paych" "github.com/filecoin-project/lotus/chain/types" @@ -56,7 +57,6 @@ func (sm *mockStateManager) setAccountAddress(a address.Address, lookup address. func (sm *mockStateManager) setPaychState(a address.Address, actor *types.Actor, state paych.State) { sm.lk.Lock() defer sm.lk.Unlock() - sm.paychState[a] = mockPchState{actor, state} } diff --git a/paychmgr/paych.go b/paychmgr/paych.go index 2cf20098677..e6799191166 100644 --- a/paychmgr/paych.go +++ b/paychmgr/paych.go @@ -539,9 +539,6 @@ func (ca *channelAccessor) laneState(state paych.State, ch address.Address) (map return nil, err } - // Note: we use a map instead of an array to store laneStates because the - // client sets the lane ID (the index) and potentially they could use a - // very large index. for _, v := range vouchers { for range v.Voucher.Merges { return nil, xerrors.Errorf("paych merges not handled yet") diff --git a/paychmgr/paych_test.go b/paychmgr/paych_test.go index 71c2bd854e6..56e7e9089a0 100644 --- a/paychmgr/paych_test.go +++ b/paychmgr/paych_test.go @@ -816,8 +816,6 @@ func testSetupMgrWithChannel(t *testing.T) *testScaffold { mock := newMockManagerAPI() mock.setAccountAddress(fromAcct, from) mock.setAccountAddress(toAcct, to) - //mock.setAccountState(fromAcct, account.State{Address: from}) - //mock.setAccountState(toAcct, account.State{Address: to}) // Create channel in state balance := big.NewInt(20) diff --git a/paychmgr/paychvoucherfunds_test.go b/paychmgr/paychvoucherfunds_test.go index 735ea54aceb..dcbb4acc9ee 100644 --- a/paychmgr/paychvoucherfunds_test.go +++ b/paychmgr/paychvoucherfunds_test.go @@ -4,13 +4,14 @@ import ( "context" "testing" + "github.com/filecoin-project/lotus/chain/actors/builtin/paych" + paychmock "github.com/filecoin-project/lotus/chain/actors/builtin/paych/mock" + "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/specs-actors/actors/builtin" - "github.com/filecoin-project/specs-actors/actors/builtin/account" - paych "github.com/filecoin-project/specs-actors/actors/builtin/paych" tutils "github.com/filecoin-project/specs-actors/support/testing" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -36,8 +37,8 @@ func TestPaychAddVoucherAfterAddFunds(t *testing.T) { defer mock.close() // Add the from signing key to the wallet - mock.setAccountState(fromAcct, account.State{Address: from}) - mock.setAccountState(toAcct, account.State{Address: to}) + mock.setAccountAddress(fromAcct, from) + mock.setAccountAddress(toAcct, to) mock.addSigningKey(fromKeyPrivate) mgr, err := newManager(store, mock) @@ -59,12 +60,7 @@ func TestPaychAddVoucherAfterAddFunds(t *testing.T) { Nonce: 0, Balance: createAmt, } - mock.setPaychState(ch, act, paych.State{ - From: fromAcct, - To: toAcct, - SettlingAt: abi.ChainEpoch(0), - MinSettleHeight: abi.ChainEpoch(0), - }) + mock.setPaychState(ch, act, paychmock.NewMockPayChState(fromAcct, toAcct, abi.ChainEpoch(0), make(map[uint64]paych.LaneState))) // Wait for create response to be processed by manager _, err = mgr.GetPaychWaitReady(ctx, createMsgCid) From 889dd3cb3fa6e44c2e590fd1908df90db9360010 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 1 Oct 2020 08:51:01 -0700 Subject: [PATCH 18/27] improve the UX for replacing messages --- cli/mpool.go | 65 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/cli/mpool.go b/cli/mpool.go index 65f4ef942c0..a8c73b656ea 100644 --- a/cli/mpool.go +++ b/cli/mpool.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" + cid "github.com/ipfs/go-cid" "github.com/urfave/cli/v2" "golang.org/x/xerrors" @@ -43,6 +44,10 @@ var mpoolPending = &cli.Command{ Name: "local", Usage: "print pending messages for addresses in local wallet only", }, + &cli.BoolFlag{ + Name: "cids", + Usage: "only print cids of messages in output", + }, }, Action: func(cctx *cli.Context) error { api, closer, err := GetFullNodeAPI(cctx) @@ -79,11 +84,15 @@ var mpoolPending = &cli.Command{ } } - out, err := json.MarshalIndent(msg, "", " ") - if err != nil { - return err + if cctx.Bool("cids") { + fmt.Println(msg.Cid()) + } else { + out, err := json.MarshalIndent(msg, "", " ") + if err != nil { + return err + } + fmt.Println(string(out)) } - fmt.Println(string(out)) } return nil @@ -308,21 +317,8 @@ var mpoolReplaceCmd = &cli.Command{ Usage: "Spend up to X FIL for this message (applicable for auto mode)", }, }, - ArgsUsage: "[from] [nonce]", + ArgsUsage: " | ", Action: func(cctx *cli.Context) error { - if cctx.Args().Len() < 2 { - return cli.ShowCommandHelp(cctx, cctx.Command.Name) - } - - from, err := address.NewFromString(cctx.Args().Get(0)) - if err != nil { - return err - } - - nonce, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) - if err != nil { - return err - } api, closer, err := GetFullNodeAPI(cctx) if err != nil { @@ -332,6 +328,39 @@ var mpoolReplaceCmd = &cli.Command{ ctx := ReqContext(cctx) + var from address.Address + var nonce uint64 + switch cctx.Args().Len() { + case 1: + mcid, err := cid.Decode(cctx.Args().First()) + if err != nil { + return err + } + + msg, err := api.ChainGetMessage(ctx, mcid) + if err != nil { + return fmt.Errorf("could not find referenced message: %w", err) + } + + from = msg.From + nonce = msg.Nonce + case 2: + f, err := address.NewFromString(cctx.Args().Get(0)) + if err != nil { + return err + } + + n, err := strconv.ParseUint(cctx.Args().Get(1), 10, 64) + if err != nil { + return err + } + + from = f + nonce = n + default: + return cli.ShowCommandHelp(cctx, cctx.Command.Name) + } + ts, err := api.ChainHead(ctx) if err != nil { return xerrors.Errorf("getting chain head: %w", err) From 9b4274638c55ee80a69520332325d91dceed5785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 20:51:36 +0200 Subject: [PATCH 19/27] pcr: fix build --- cmd/lotus-pcr/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 1545ec2245d..70568a17abc 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -564,10 +564,10 @@ type refunderNodeApi interface { ChainGetTipSetByHeight(ctx context.Context, epoch abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) StateMinerInitialPledgeCollateral(ctx context.Context, addr address.Address, precommitInfo miner.SectorPreCommitInfo, tsk types.TipSetKey) (types.BigInt, error) - StateMinerInfo(context.Context, address.Address, types.TipSetKey) (api.MinerInfo, error) + StateMinerInfo(context.Context, address.Address, types.TipSetKey) (miner.MinerInfo, error) StateSectorPreCommitInfo(ctx context.Context, addr address.Address, sector abi.SectorNumber, tsk types.TipSetKey) (miner.SectorPreCommitOnChainInfo, error) StateMinerAvailableBalance(context.Context, address.Address, types.TipSetKey) (types.BigInt, error) - StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, filterOut bool, tsk types.TipSetKey) ([]*api.ChainSectorInfo, error) + StateMinerSectors(ctx context.Context, addr address.Address, filter *bitfield.BitField, tsk types.TipSetKey) ([]*miner.SectorOnChainInfo, error) StateMinerFaults(ctx context.Context, addr address.Address, tsk types.TipSetKey) (bitfield.BitField, error) StateListMiners(context.Context, types.TipSetKey) ([]address.Address, error) StateGetActor(ctx context.Context, actor address.Address, tsk types.TipSetKey) (*types.Actor, error) From 078044153f598e486b37c9044f52fc376e15e2cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 1 Oct 2020 21:07:39 +0200 Subject: [PATCH 20/27] pcr: Appease the linter --- cmd/lotus-pcr/main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/lotus-pcr/main.go b/cmd/lotus-pcr/main.go index 70568a17abc..4ce5bbb9f95 100644 --- a/cmd/lotus-pcr/main.go +++ b/cmd/lotus-pcr/main.go @@ -615,6 +615,10 @@ func (r *refunder) FindMiners(ctx context.Context, tipset *types.TipSet, refunds // Look up and find all addresses associated with the miner minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } allAddresses := []address.Address{} @@ -673,7 +677,7 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet return nil, err } - defer f.Close() + defer f.Close() // nolint:errcheck w = bufio.NewWriter(f) } @@ -703,6 +707,10 @@ func (r *refunder) EnsureMinerMinimums(ctx context.Context, tipset *types.TipSet // Look up and find all addresses associated with the miner minerInfo, err := r.api.StateMinerInfo(ctx, maddr, tipset.Key()) + if err != nil { + log.Errorw("failed", "err", err, "height", tipset.Height(), "key", tipset.Key(), "miner", maddr) + continue + } allAddresses := []address.Address{minerInfo.Worker, minerInfo.Owner} allAddresses = append(allAddresses, minerInfo.ControlAddresses...) From 8fe8a5df454a3dff2ce06ce542760724dde21477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 18:34:50 +0200 Subject: [PATCH 21/27] Cap market provider messages --- markets/storageadapter/provider.go | 30 ++++++++++++++++++++---------- node/builder.go | 4 +++- node/config/def.go | 16 ++++++++++------ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/markets/storageadapter/provider.go b/markets/storageadapter/provider.go index 9f610d76a05..b1071adcd1d 100644 --- a/markets/storageadapter/provider.go +++ b/markets/storageadapter/provider.go @@ -25,15 +25,15 @@ import ( "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/build" "github.com/filecoin-project/lotus/chain/actors" - "github.com/filecoin-project/lotus/chain/events" - "github.com/filecoin-project/lotus/chain/actors/builtin/market" "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/events/state" "github.com/filecoin-project/lotus/chain/types" sealing "github.com/filecoin-project/lotus/extern/storage-sealing" "github.com/filecoin-project/lotus/lib/sigs" "github.com/filecoin-project/lotus/markets/utils" + "github.com/filecoin-project/lotus/node/config" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/lotus/storage/sectorblocks" ) @@ -50,14 +50,24 @@ type ProviderNodeAdapter struct { secb *sectorblocks.SectorBlocks ev *events.Events + + publishSpec, addBalanceSpec *api.MessageSendSpec } -func NewProviderNodeAdapter(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { - return &ProviderNodeAdapter{ - FullNode: full, - dag: dag, - secb: secb, - ev: events.NewEvents(context.TODO(), full), +func NewProviderNodeAdapter(fc *config.MinerFeeConfig) func(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { + return func(dag dtypes.StagingDAG, secb *sectorblocks.SectorBlocks, full api.FullNode) storagemarket.StorageProviderNode { + na := &ProviderNodeAdapter{ + FullNode: full, + + dag: dag, + secb: secb, + ev: events.NewEvents(context.TODO(), full), + } + if fc != nil { + na.publishSpec = &api.MessageSendSpec{MaxFee: abi.TokenAmount(fc.MaxPublishDealsFee)} + na.addBalanceSpec = &api.MessageSendSpec{MaxFee: abi.TokenAmount(fc.MaxMarketBalanceAddFee)} + } + return na } } @@ -84,7 +94,7 @@ func (n *ProviderNodeAdapter) PublishDeals(ctx context.Context, deal storagemark Value: types.NewInt(0), Method: builtin0.MethodsMarket.PublishStorageDeals, Params: params, - }, nil) + }, n.publishSpec) if err != nil { return cid.Undef, err } @@ -183,7 +193,7 @@ func (n *ProviderNodeAdapter) AddFunds(ctx context.Context, addr address.Address From: addr, Value: amount, Method: builtin0.MethodsMarket.AddBalance, - }, nil) + }, n.addBalanceSpec) if err != nil { return cid.Undef, err } diff --git a/node/builder.go b/node/builder.go index 8512d0a0c4c..41fb01ef7c8 100644 --- a/node/builder.go +++ b/node/builder.go @@ -344,7 +344,7 @@ func Online() Option { Override(new(dtypes.DealFilter), modules.BasicDealFilter(nil)), Override(new(modules.ProviderDealFunds), modules.NewProviderDealFunds), Override(new(storagemarket.StorageProvider), modules.StorageProvider), - Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(nil)), Override(HandleRetrievalKey, modules.HandleRetrieval), Override(GetParamsKey, modules.GetParams), Override(HandleDealsKey, modules.HandleDeals), @@ -463,6 +463,8 @@ func ConfigStorageMiner(c interface{}) Option { Override(new(dtypes.DealFilter), modules.BasicDealFilter(dealfilter.CliDealFilter(cfg.Dealmaking.Filter))), ), + Override(new(storagemarket.StorageProviderNode), storageadapter.NewProviderNodeAdapter(&cfg.Fees)), + Override(new(sectorstorage.SealerConfig), cfg.Storage), Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)), ) diff --git a/node/config/def.go b/node/config/def.go index 9fee8895af5..420f8b99029 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -61,9 +61,11 @@ type SealingConfig struct { } type MinerFeeConfig struct { - MaxPreCommitGasFee types.FIL - MaxCommitGasFee types.FIL - MaxWindowPoStGasFee types.FIL + MaxPreCommitGasFee types.FIL + MaxCommitGasFee types.FIL + MaxWindowPoStGasFee types.FIL + MaxPublishDealsFee types.FIL + MaxMarketBalanceAddFee types.FIL } // API contains configs for API endpoint @@ -173,9 +175,11 @@ func DefaultStorageMiner() *StorageMiner { }, Fees: MinerFeeConfig{ - MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05 - MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), - MaxWindowPoStGasFee: types.FIL(types.FromFil(50)), + MaxPreCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), // 0.05 + MaxCommitGasFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(20))), + MaxWindowPoStGasFee: types.FIL(types.FromFil(50)), + MaxPublishDealsFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(33))), // 0.03ish + MaxMarketBalanceAddFee: types.FIL(types.BigDiv(types.FromFil(1), types.NewInt(100))), // 0.01 }, } cfg.Common.API.ListenAddress = "/ip4/127.0.0.1/tcp/2345/http" From e848c13ff1f25cc0bb610bfbae8c7c206b9ce570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 19:31:21 +0200 Subject: [PATCH 22/27] client: bump default deal start buffer --- node/impl/client/client.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/impl/client/client.go b/node/impl/client/client.go index 7cb087ec7b3..39f0ab19ad3 100644 --- a/node/impl/client/client.go +++ b/node/impl/client/client.go @@ -60,7 +60,7 @@ import ( var DefaultHashFunction = uint64(mh.BLAKE2B_MIN + 31) -const dealStartBufferHours uint64 = 24 +const dealStartBufferHours uint64 = 49 type API struct { fx.In @@ -154,7 +154,7 @@ func (a *API) ClientStartDeal(ctx context.Context, params *api.StartDealParams) } blocksPerHour := 60 * 60 / build.BlockDelaySecs - dealStart = ts.Height() + abi.ChainEpoch(dealStartBufferHours*blocksPerHour) + dealStart = ts.Height() + abi.ChainEpoch(dealStartBufferHours*blocksPerHour) // TODO: Get this from storage ask } result, err := a.SMDealClient.ProposeStorageDeal(ctx, storagemarket.ProposeStorageDealParams{ From 2c1ef3708eeccf67dd0361f9d48d4aa27f8d10c2 Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 2 Oct 2020 12:16:59 -0700 Subject: [PATCH 23/27] Add verified flag to interactive deal creation --- cli/client.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/cli/client.go b/cli/client.go index 255f5a70c74..6ba143753e4 100644 --- a/cli/client.go +++ b/cli/client.go @@ -12,13 +12,12 @@ import ( "text/tabwriter" "time" - "github.com/filecoin-project/specs-actors/actors/builtin" - tm "github.com/buger/goterm" "github.com/docker/go-units" "github.com/fatih/color" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-fil-markets/retrievalmarket" + "github.com/filecoin-project/specs-actors/actors/builtin" "github.com/ipfs/go-cid" "github.com/ipfs/go-cidutil/cidenc" "github.com/libp2p/go-libp2p-core/peer" @@ -476,6 +475,7 @@ func interactiveDeal(cctx *cli.Context) error { var ask storagemarket.StorageAsk var epochPrice big.Int var epochs abi.ChainEpoch + var verified bool var a address.Address if from := cctx.String("from"); from != "" { @@ -572,6 +572,53 @@ func interactiveDeal(cctx *cli.Context) error { ask = *a // TODO: run more validation + state = "verified" + case "verified": + ts, err := api.ChainHead(ctx) + if err != nil { + return err + } + + dcap, err := api.StateVerifiedClientStatus(ctx, a, ts.Key()) + if err != nil { + return err + } + + if dcap == nil { + state = "confirm" + continue + } + + color.Blue(".. checking verified deal eligibility\n") + ds, err := api.ClientDealSize(ctx, data) + if err != nil { + return err + } + + if dcap.Uint64() < uint64(ds.PieceSize) { + color.Yellow(".. not enough DataCap available for a verified deal\n") + state = "confirm" + continue + } + + fmt.Print("\nMake this a verified deal? (yes/no): ") + + var yn string + _, err = fmt.Scan(&yn) + if err != nil { + return err + } + + switch yn { + case "yes": + verified = true + case "no": + verified = false + default: + fmt.Println("Type in full 'yes' or 'no'") + continue + } + state = "confirm" case "confirm": fromBal, err := api.WalletBalance(ctx, a) @@ -603,6 +650,7 @@ func interactiveDeal(cctx *cli.Context) error { fmt.Printf("Piece size: %s (Payload size: %s)\n", units.BytesSize(float64(ds.PieceSize)), units.BytesSize(float64(ds.PayloadSize))) fmt.Printf("Duration: %s\n", dur) fmt.Printf("Total price: ~%s (%s per epoch)\n", types.FIL(totalPrice), types.FIL(epochPrice)) + fmt.Printf("Verified: %v\n", verified) state = "accept" case "accept": @@ -637,7 +685,7 @@ func interactiveDeal(cctx *cli.Context) error { MinBlocksDuration: uint64(epochs), DealStartEpoch: abi.ChainEpoch(cctx.Int64("start-epoch")), FastRetrieval: cctx.Bool("fast-retrieval"), - VerifiedDeal: false, // TODO: Allow setting + VerifiedDeal: verified, }) if err != nil { return err From 73b9fe36e3e24cedf027de51796529fdd1cf7aae Mon Sep 17 00:00:00 2001 From: Ingar Shu Date: Fri, 2 Oct 2020 13:07:40 -0700 Subject: [PATCH 24/27] Use verified price for verified deals --- cli/client.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cli/client.go b/cli/client.go index 6ba143753e4..7494815bf35 100644 --- a/cli/client.go +++ b/cli/client.go @@ -637,10 +637,15 @@ func interactiveDeal(cctx *cli.Context) error { epochs = abi.ChainEpoch(dur / (time.Duration(build.BlockDelaySecs) * time.Second)) // TODO: do some more or epochs math (round to miner PP, deal start buffer) + pricePerGib := ask.Price + if verified { + pricePerGib = ask.VerifiedPrice + } + gib := types.NewInt(1 << 30) // TODO: price is based on PaddedPieceSize, right? - epochPrice = types.BigDiv(types.BigMul(ask.Price, types.NewInt(uint64(ds.PieceSize))), gib) + epochPrice = types.BigDiv(types.BigMul(pricePerGib, types.NewInt(uint64(ds.PieceSize))), gib) totalPrice := types.BigMul(epochPrice, types.NewInt(uint64(epochs))) fmt.Printf("-----\n") From db91d22a8a53c5b2cc8d976ec270f0f553c9799b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 23:14:47 +0200 Subject: [PATCH 25/27] docsgen --- documentation/en/api-methods.md | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/documentation/en/api-methods.md b/documentation/en/api-methods.md index 2b28816f7a2..19774802e96 100644 --- a/documentation/en/api-methods.md +++ b/documentation/en/api-methods.md @@ -72,6 +72,7 @@ * [MpoolPending](#MpoolPending) * [MpoolPush](#MpoolPush) * [MpoolPushMessage](#MpoolPushMessage) + * [MpoolPushUntrusted](#MpoolPushUntrusted) * [MpoolSelect](#MpoolSelect) * [MpoolSetConfig](#MpoolSetConfig) * [MpoolSub](#MpoolSub) @@ -1779,6 +1780,43 @@ Response: } ``` +### MpoolPushUntrusted +MpoolPushUntrusted pushes a signed message to mempool from untrusted sources. + + +Perms: write + +Inputs: +```json +[ + { + "Message": { + "Version": 42, + "To": "t01234", + "From": "t01234", + "Nonce": 42, + "Value": "0", + "GasLimit": 9, + "GasFeeCap": "0", + "GasPremium": "0", + "Method": 1, + "Params": "Ynl0ZSBhcnJheQ==" + }, + "Signature": { + "Type": 2, + "Data": "Ynl0ZSBhcnJheQ==" + } + } +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + ### MpoolSelect MpoolSelect returns a list of pending messages for inclusion in the next block From 0591bac768ca241506cfd139147ebdd14da2e263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 23:18:37 +0200 Subject: [PATCH 26/27] Compare more accurately Co-authored-by: dirkmc --- chain/messagepool/messagepool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chain/messagepool/messagepool.go b/chain/messagepool/messagepool.go index d9a2b75f4a0..83aa5c6b784 100644 --- a/chain/messagepool/messagepool.go +++ b/chain/messagepool/messagepool.go @@ -251,7 +251,7 @@ func (ms *msgSet) add(m *types.SignedMessage, mp *MessagePool, strict, untrusted //ms.requiredFunds.Sub(ms.requiredFunds, exms.Message.Value.Int) } - if !has && strict && len(ms.msgs) > maxActorPendingMessages { + if !has && strict && len(ms.msgs) >= maxActorPendingMessages { log.Errorf("too many pending messages from actor %s", m.Message.From) return false, ErrTooManyPendingMessages } From 1319d7072ecf9094e6515e2de4bb8e41743a5948 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 2 Oct 2020 23:38:54 +0200 Subject: [PATCH 27/27] shed: Fix lint --- cmd/lotus-shed/pruning.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/lotus-shed/pruning.go b/cmd/lotus-shed/pruning.go index 6f0c20541b5..79158c3a308 100644 --- a/cmd/lotus-shed/pruning.go +++ b/cmd/lotus-shed/pruning.go @@ -137,13 +137,13 @@ var stateTreePruneCmd = &cli.Command{ return err } - defer ds.Close() + defer ds.Close() //nolint:errcheck mds, err := lkrepo.Datastore("/metadata") if err != nil { return err } - defer mds.Close() + defer mds.Close() //nolint:errcheck if cctx.Bool("only-ds-gc") { gcds, ok := ds.(datastore.GCDatastore) @@ -156,9 +156,8 @@ var stateTreePruneCmd = &cli.Command{ } fmt.Println("gc complete!") return nil - } else { - return fmt.Errorf("datastore doesnt support gc") } + return fmt.Errorf("datastore doesnt support gc") } bs := blockstore.NewBlockstore(ds) @@ -194,7 +193,7 @@ var stateTreePruneCmd = &cli.Command{ } fmt.Println() - fmt.Printf("Succesfully marked keep set! (%d objects)\n", goodSet.Len()) + fmt.Printf("Successfully marked keep set! (%d objects)\n", goodSet.Len()) if cctx.Bool("dry-run") { return nil