Merge branch 'master' into feat/fvm

CHANGELOG.md

@@ -1,5 +1,106 @@
 # Lotus changelog
 
+# 1.14.2 / 2022-02-24
+
+This is an **optional** release of lotus, that's had a couple more improvements w.r.t Snap experience for storage providers in preparation of the[upcoming OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550). 
+
+Note that the network is STILL scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to at least Lotus v1.14.0 before that time. Storage providers must update their daemons, miners, and worker(s).
+
+Wanna know how to Snap your deal? Check [this](https://github.com/filecoin-project/lotus/discussions/8141) out! 
+
+## Bug Fixes
+- fix lotus-bench for sealing jobs (#8173)
+- fix:sealing:really-do-it flag for abort upgrade (#8181)
+- fix:proving:post check sector handles snap deals replica faults (#8177)
+- fix: sealing: missing file type (#8180)
+
+## Others
+- Retract force-pushed v1.14.0 to work around stale gomod caches (#8159): We originally tagged v1.14.0 off the wrong 
+  commit and fixed that by a force push, in which is a really bad practise since it messes up the go mod. Therefore, 
+  we want to retract it and users may use v1.14.1&^.
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| @zenground0 | 2 | +73/-58 | 12 |
+| @eben.xie | 1 | +7/-0 | 1 |
+| @jennijuju | 1 | +4/-0 | 1 |
+| @jennijuju | 1 | +2/-1 | 1 |
+| @ribasushi | 1 | +2/-0 | 1 |
+
+# 1.14.1 / 2022-02-18
+
+This is an **optional** release of lotus, that fixes the incorrect *comment* of network v15 OhSnap upgrade **date**. Note the actual upgrade epoch in [v1.14.0](https://github.com/filecoin-project/lotus/releases/tag/v1.14.0) was correct.
+
+# 1.14.0 / 2022-02-17
+
+This is a MANDATORY release of Lotus that introduces [Filecoin network v15, 
+codenamed the OhSnap upgrade](https://github.com/filecoin-project/community/discussions/74?sort=new#discussioncomment-1922550).
+
+The network is scheduled to upgrade to v15 on March 1st at 2022-03-01T15:00:00Z. All node operators, including storage providers, must upgrade to this release (or a later release) before that time. Storage providers must update their daemons, miners, and worker(s).
+
+The OhSnap upgrade introduces the following FIPs, delivered in [actors v7](https://github.com/filecoin-project/specs-actors/releases/tag/v7.0.0):
+- [FIP-0019 Snap Deals](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0019.md)
+- [FIP-0028 Remove Datacap from Verified clients](https://github.com/filecoin-project/FIPs/pull/226)
+
+It is recommended that storage providers download the new params before updating their node, miner, and workers. To do so:
+
+- Download Lotus v1.14.0 or later
+- run `make lotus-shed`
+- run `./lotus-shed fetch-params` with the appropriate `proving-params` flag
+- Upgrade the Lotus daemon and miner **when the previous step is complete**
+
+All node operators, including storage providers, should be aware that a pre-migration will begin at 2022-03-01T13:30:00Z (90 minutes before the real upgrade). The pre-migration will take between 20 and 50 minutes, depending on hardware specs. During this time, expect slower block validation times, increased CPU and memory usage, and longer delays for API queries.
+
+## New Features and Changes
+- Integrate actor v7-rc1:
+  - Integrate v7 actors ([#7617](https://github.com/filecoin-project/lotus/pull/7617))
+  - feat: state: Fast migration for v15 ([#7933](https://github.com/filecoin-project/lotus/pull/7933))
+  - fix: blockstore: Add missing locks to autobatch::Get() [#7939](https://github.com/filecoin-project/lotus/pull/7939))
+  - correctness fixes for the autobatch blockstore ([#7940](https://github.com/filecoin-project/lotus/pull/7940))
+- Implement and support [FIP-0019 Snap Deals](https://github.com/filecoin-project/FIPs/blob/master/FIPS/fip-0019.md)
+  - chore: deps: Integrate proof v11.0.0 ([#7923](https://github.com/filecoin-project/lotus/pull/7923))
+  - Snap Deals Lotus Integration: FSM Posting and integration test ([#7810](https://github.com/filecoin-project/lotus/pull/7810))
+  - Feat/sector storage unseal ([#7730](https://github.com/filecoin-project/lotus/pull/7730))
+  - Feat/snap deals storage ([#7615](https://github.com/filecoin-project/lotus/pull/7615))
+  - fix: sealing: Add more deal expiration checks during PRU pipeline ([#7871](https://github.com/filecoin-project/lotus/pull/7871))
+  - chore: deps: Update go-paramfetch ([#7917](https://github.com/filecoin-project/lotus/pull/7917))
+  - feat: #7880 gas: add gas charge for VerifyReplicaUpdate ([#7897](https://github.com/filecoin-project/lotus/pull/7897))
+  - enhancement: sectors: disable existing cc upgrade path 2 days before the upgrade epoch ([#7900](https://github.com/filecoin-project/lotus/pull/7900))
+
+## Improvements
+- updating to new datastore/blockstore code with contexts ([#7646](https://github.com/filecoin-project/lotus/pull/7646))
+- reorder transfer checks so as to ensure sending 2B FIL to yourself fails if you don't have that amount ([#7637](https://github.com/filecoin-project/lotus/pull/7637))
+- VM: Circ supply should be constant per epoch ([#7811](https://github.com/filecoin-project/lotus/pull/7811))
+
+## Bug Fixes
+- Fix: state: circsuypply calc around null blocks ([#7890](https://github.com/filecoin-project/lotus/pull/7890))
+- Mempool msg selection should respect block message limits ([#7321](https://github.com/filecoin-project/lotus/pull/7321))
+  SplitStore: supress compaction near upgrades ([#7734](https://github.com/filecoin-project/lotus/pull/7734))
+
+## Others
+- chore: create pull_request_template.md ([#7726](https://github.com/filecoin-project/lotus/pull/7726))
+
+## Contributors
+
+| Contributor | Commits | Lines ± | Files Changed |
+|-------------|---------|---------|---------------|
+| Aayush Rajasekaran | 41 | +5538/-1205 | 189 |
+| zenground0 | 11 | +3316/-524 | 124 |
+| Jennifer Wang | 29 | +714/-599 | 68 |
+| ZenGround0 | 3 | +263/-25 | 11 |
+| c r | 2 | +198/-30 | 6 |
+| vyzo | 4 | +189/-7 | 7 |
+| Aayush | 11 | +146/-48 | 49 |
+| web3-bot | 10 | +99/-17 | 10 |
+| Steven Allen | 1 | +55/-37 | 1 |
+| Jiaying Wang | 5 | +30/-8 | 5 |
+| Jakub Sztandera | 2 | +8/-3 | 3 |
+| Łukasz Magiera | 1 | +3/-3 | 2 |
+| Travis Person | 1 | +2/-2 | 2 |
+| Rod Vagg | 1 | +2/-2 | 2 |
+
 # v1.13.2 /  2022-01-09
 
 Lotus v1.13.2 is a *highly recommended* feature release with remarkable retrieval improvements, new features like 

api/api_gateway.go

@@ -45,8 +45,9 @@ type Gateway interface {
 	GasEstimateMessageGas(ctx context.Context, msg *types.Message, spec *MessageSendSpec, tsk types.TipSetKey) (*types.Message, error)
 	MpoolPush(ctx context.Context, sm *types.SignedMessage) (cid.Cid, error)
 	MsigGetAvailableBalance(ctx context.Context, addr address.Address, tsk types.TipSetKey) (types.BigInt, error)
-	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
 	MsigGetPending(context.Context, address.Address, types.TipSetKey) ([]*MsigTransaction, error)
+	MsigGetVested(ctx context.Context, addr address.Address, start types.TipSetKey, end types.TipSetKey) (types.BigInt, error)
+	MsigGetVestingSchedule(ctx context.Context, addr address.Address, tsk types.TipSetKey) (MsigVesting, error)
 	StateAccountKey(ctx context.Context, addr address.Address, tsk types.TipSetKey) (address.Address, error)
 	StateDealProviderCollateralBounds(ctx context.Context, size abi.PaddedPieceSize, verified bool, tsk types.TipSetKey) (DealCollateralBounds, error)
 	StateGetActor(ctx context.Context, actor address.Address, ts types.TipSetKey) (*types.Actor, error)

api/api_storage.go

@@ -256,7 +256,7 @@ type StorageMiner interface {
 	// the path specified when calling CreateBackup is within the base path
 	CreateBackup(ctx context.Context, fpath string) error //perm:admin
 
-	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
+	CheckProvable(ctx context.Context, pp abi.RegisteredPoStProof, sectors []storage.SectorRef, update []bool, expensive bool) (map[abi.SectorNumber]string, error) //perm:admin
 
 	ComputeProof(ctx context.Context, ssi []builtin.ExtendedSectorInfo, rand abi.PoStRandomness, poStEpoch abi.ChainEpoch, nv abinetwork.Version) ([]builtin.PoStProof, error) //perm:read
 }

api/version.go

@@ -57,7 +57,7 @@ var (
 	FullAPIVersion0 = newVer(1, 5, 0)
 	FullAPIVersion1 = newVer(2, 2, 0)
 
-	MinerAPIVersion0  = newVer(1, 3, 0)
+	MinerAPIVersion0  = newVer(1, 4, 0)
 	WorkerAPIVersion0 = newVer(1, 5, 0)
 )
 

blockstore/splitstore/splitstore_reify.go

@@ -1,6 +1,7 @@
 package splitstore
 
 import (
+	"errors"
 	"runtime"
 	"sync/atomic"
 
@@ -10,13 +11,12 @@ import (
 	cid "github.com/ipfs/go-cid"
 )
 
-var EnableReification = false
+var (
+	errReifyLimit = errors.New("reification limit reached")
+	ReifyLimit    = 16384
+)
 
 func (s *SplitStore) reifyColdObject(c cid.Cid) {
-	if !EnableReification {
-		return
-	}
-
 	if !s.isWarm() {
 		return
 	}
@@ -104,12 +104,18 @@ func (s *SplitStore) doReify(c cid.Cid) {
 	s.txnLk.RLock()
 	defer s.txnLk.RUnlock()
 
+	count := 0
 	err := s.walkObjectIncomplete(c, newTmpVisitor(),
 		func(c cid.Cid) error {
 			if isUnitaryObject(c) {
 				return errStopWalk
 			}
 
+			count++
+			if count > ReifyLimit {
+				return errReifyLimit
+			}
+
 			s.reifyMx.Lock()
 			_, inProgress := s.reifyInProgress[c]
 			if !inProgress {
@@ -150,6 +156,11 @@ func (s *SplitStore) doReify(c cid.Cid) {
 		})
 
 	if err != nil {
+		if xerrors.Is(err, errReifyLimit) {
+			log.Debug("reification aborted; reify limit reached")
+			return
+		}
+
 		log.Warnf("error walking cold object for reification (cid: %s): %s", c, err)
 		return
 	}

blockstore/splitstore/splitstore_test.go

@@ -495,8 +495,102 @@ func testSplitStoreReification(t *testing.T, f func(context.Context, blockstore.
 	}
 }
 
+func testSplitStoreReificationLimit(t *testing.T, f func(context.Context, blockstore.Blockstore, cid.Cid) error) {
+	ds := dssync.MutexWrap(datastore.NewMapDatastore())
+	hot := newMockStore()
+	cold := newMockStore()
+
+	mkRandomBlock := func() blocks.Block {
+		data := make([]byte, 128)
+		_, err := rand.Read(data)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		return blocks.NewBlock(data)
+	}
+
+	block1 := mkRandomBlock()
+	block2 := mkRandomBlock()
+	block3 := mkRandomBlock()
+
+	hdr := mock.MkBlock(nil, 0, 0)
+	hdr.Messages = block1.Cid()
+	hdr.ParentMessageReceipts = block2.Cid()
+	hdr.ParentStateRoot = block3.Cid()
+	block4, err := hdr.ToStorageBlock()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	allBlocks := []blocks.Block{block1, block2, block3, block4}
+	for _, blk := range allBlocks {
+		err := cold.Put(context.Background(), blk)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	path, err := ioutil.TempDir("", "splitstore.*")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Cleanup(func() {
+		_ = os.RemoveAll(path)
+	})
+
+	ss, err := Open(path, ds, hot, cold, &Config{MarkSetType: "map"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ss.Close() //nolint
+
+	ss.warmupEpoch = 1
+	go ss.reifyOrchestrator()
+
+	waitForReification := func() {
+		for {
+			ss.reifyMx.Lock()
+			ready := len(ss.reifyPend) == 0 && len(ss.reifyInProgress) == 0
+			ss.reifyMx.Unlock()
+
+			if ready {
+				return
+			}
+
+			time.Sleep(time.Millisecond)
+		}
+	}
+
+	// do a hot access -- nothing should be reified as the limit should be exceeded
+	oldReifyLimit := ReifyLimit
+	ReifyLimit = 2
+	t.Cleanup(func() {
+		ReifyLimit = oldReifyLimit
+	})
+
+	err = f(blockstore.WithHotView(context.Background()), ss, block4.Cid())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	waitForReification()
+
+	for _, blk := range allBlocks {
+		has, err := hot.Has(context.Background(), blk.Cid())
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if has {
+			t.Fatal("block unexpectedly reified")
+		}
+	}
+
+}
+
 func TestSplitStoreReification(t *testing.T) {
-	EnableReification = true
 	t.Log("test reification with Has")
 	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
 		_, err := s.Has(ctx, c)
@@ -516,6 +610,11 @@ func TestSplitStoreReification(t *testing.T) {
 	testSplitStoreReification(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
 		return s.View(ctx, c, func(_ []byte) error { return nil })
 	})
+	t.Log("test reification limit")
+	testSplitStoreReificationLimit(t, func(ctx context.Context, s blockstore.Blockstore, c cid.Cid) error {
+		_, err := s.Has(ctx, c)
+		return err
+	})
 }
 
 type mockChain struct {

blockstore/splitstore/visitor.go

@@ -26,6 +26,10 @@ type tmpVisitor struct {
 var _ ObjectVisitor = (*tmpVisitor)(nil)
 
 func (v *tmpVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
 	return v.set.Visit(c), nil
 }
 
@@ -45,6 +49,10 @@ func newConcurrentVisitor() *concurrentVisitor {
 }
 
 func (v *concurrentVisitor) Visit(c cid.Cid) (bool, error) {
+	if isUnitaryObject(c) {
+		return false, nil
+	}
+
 	v.mx.Lock()
 	defer v.mx.Unlock()
 

build/params_mainnet.go

@@ -67,7 +67,8 @@ const UpgradeHyperdriveHeight = 892800
 // 2021-10-26T13:30:00Z
 const UpgradeChocolateHeight = 1231620
 
-var UpgradeOhSnapHeight = abi.ChainEpoch(999999999999)
+// 2022-03-01T15:00:00Z
+var UpgradeOhSnapHeight = abi.ChainEpoch(1594680)
 
 func init() {
 	if os.Getenv("LOTUS_USE_TEST_ADDRESSES") != "1" {