feat(f3): checkpoint tipsets that are finalized by F3 (#12460)

* Checkpoint tipsets that are finalized by F3

Once a decision is received from F3, checkpoint it in chain-store.

As part of checkpointing, refactor the F3 tipset wrapper to reduce type
casting.

Note that go-f3 module now uses Go 1.22, which requires Lotus go module
to be updated accordingly.

Part of filecoin-project/go-f3#603

* Make checkpointing chainstore optional via build constraints

CHANGELOG.md

@@ -9,6 +9,7 @@
 ## New features
 
 * Add `EthSendRawTransactionUntrusted` RPC method to be used for the gateway when accepting `EthSendRawTransaction` and `eth_sendRawTransaction`. Applies a tighter limit on the number of messages in the queue from a single sender and applies additional restrictions on nonce increments. ([filecoin-project/lotus#12431](https://github.com/filecoin-project/lotus/pull/12431))
+* [Checkpoint TipSets finalized by F3](https://github.com/filecoin-project/lotus/pull/12460): Once a decision is made by F3, the TipSet is check-pointed in `ChainStore`. As part of this change, any missing TipSets are asynchronously synced as required by the `ChainStore` checkpointing mechanism. 
 
 ## Improvements
 

build/buildconstants/params_2k.go

@@ -200,3 +200,7 @@ var F3Enabled = true
 const ManifestServerID = "12D3KooWHcNBkqXEBrsjoveQvj6zDF3vK5S9tAfqyYaQF1LGSJwG"
 
 var F3BootstrapEpoch abi.ChainEpoch = 1000
+
+// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This
+// flag has no effect if F3 is not enabled.
+const F3Consensus = true

build/buildconstants/params_butterfly.go

@@ -101,3 +101,7 @@ var WhitelistedBlock = cid.Undef
 const F3Enabled = true
 const ManifestServerID = "12D3KooWJr9jy4ngtJNR7JC1xgLFra3DjEtyxskRYWvBK9TC3Yn6"
 const F3BootstrapEpoch abi.ChainEpoch = 1000
+
+// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This
+// flag has no effect if F3 is not enabled.
+const F3Consensus = true

build/buildconstants/params_calibnet.go

@@ -148,3 +148,7 @@ var WhitelistedBlock = cid.Undef
 const F3Enabled = true
 const ManifestServerID = "12D3KooWS9vD9uwm8u2uPyJV32QBAhKAmPYwmziAgr3Xzk2FU1Mr"
 const F3BootstrapEpoch abi.ChainEpoch = UpgradeWaffleHeight + 100
+
+// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This
+// flag has no effect if F3 is not enabled.
+const F3Consensus = true

build/buildconstants/params_interop.go

@@ -139,3 +139,7 @@ var WhitelistedBlock = cid.Undef
 const F3Enabled = true
 const ManifestServerID = "12D3KooWQJ2rdVnG4okDUB6yHQhAjNutGNemcM7XzqC9Eo4z9Jce"
 const F3BootstrapEpoch abi.ChainEpoch = 1000
+
+// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This
+// flag has no effect if F3 is not enabled.
+const F3Consensus = true

build/buildconstants/params_mainnet.go

@@ -173,3 +173,7 @@ var WhitelistedBlock = cid.MustParse("bafy2bzaceapyg2uyzk7vueh3xccxkuwbz3nxewjyg
 const F3Enabled = true
 const ManifestServerID = "12D3KooWENMwUF9YxvQxar7uBWJtZkA6amvK4xWmKXfSiHUo2Qq7"
 const F3BootstrapEpoch abi.ChainEpoch = -1
+
+// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This
+// flag has no effect if F3 is not enabled.
+const F3Consensus = false

build/buildconstants/params_testground.go

@@ -126,6 +126,10 @@ var (
 	F3Enabled                        = false
 	ManifestServerID                 = ""
 	F3BootstrapEpoch  abi.ChainEpoch = -1
+
+	// F3Consensus set whether F3 should checkpoint tipsets finalized by F3. This
+	// flag has no effect if F3 is not enabled.
+	F3Consensus = true
 )
 
 func init() {

build/openrpc/full.json

@@ -6485,9 +6485,7 @@
                                         "type": "string"
                                     },
                                     "PowerTable": {
-                                        "media": {
-                                            "binaryEncoding": "base64"
-                                        },
+                                        "title": "Content Identifier",
                                         "type": "string"
                                     }
                                 },
@@ -6548,9 +6546,7 @@
                                     "type": "array"
                                 },
                                 "PowerTable": {
-                                    "media": {
-                                        "binaryEncoding": "base64"
-                                    },
+                                    "title": "Content Identifier",
                                     "type": "string"
                                 }
                             },
@@ -6822,9 +6818,7 @@
                                         "type": "string"
                                     },
                                     "PowerTable": {
-                                        "media": {
-                                            "binaryEncoding": "base64"
-                                        },
+                                        "title": "Content Identifier",
                                         "type": "string"
                                     }
                                 },
@@ -6885,9 +6879,7 @@
                                     "type": "array"
                                 },
                                 "PowerTable": {
-                                    "media": {
-                                        "binaryEncoding": "base64"
-                                    },
+                                    "title": "Content Identifier",
                                     "type": "string"
                                 }
                             },

chain/lf3/ec.go

@@ -12,6 +12,7 @@ import (
 	"github.com/filecoin-project/go-f3/gpbft"
 	"github.com/filecoin-project/go-state-types/abi"
 
+	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
 	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/stmgr"
@@ -20,52 +21,57 @@ import (
 	"github.com/filecoin-project/lotus/chain/vm"
 )
 
+var (
+	_ ec.Backend = (*ecWrapper)(nil)
+	_ ec.TipSet  = (*f3TipSet)(nil)
+)
+
 type ecWrapper struct {
 	ChainStore   *store.ChainStore
+	Syncer       *chain.Syncer
 	StateManager *stmgr.StateManager
-}
-
-var _ ec.TipSet = (*f3TipSet)(nil)
-
-type f3TipSet types.TipSet
 
-func (ts *f3TipSet) cast() *types.TipSet {
-	return (*types.TipSet)(ts)
+	// Checkpoint sets whether to checkpoint tipsets finalized by F3 in ChainStore.
+	Checkpoint bool
 }
 
-func (ts *f3TipSet) String() string {
-	return ts.cast().String()
+type f3TipSet struct {
+	*types.TipSet
 }
 
-func (ts *f3TipSet) Key() gpbft.TipSetKey {
-	return ts.cast().Key().Bytes()
+func (ts *f3TipSet) String() string       { return ts.TipSet.String() }
+func (ts *f3TipSet) Key() gpbft.TipSetKey { return ts.TipSet.Key().Bytes() }
+func (ts *f3TipSet) Epoch() int64         { return int64(ts.TipSet.Height()) }
+
+func (ts *f3TipSet) FirstBlockHeader() *types.BlockHeader {
+	if ts.TipSet == nil || len(ts.TipSet.Blocks()) == 0 {
+		return nil
+	}
+	return ts.TipSet.Blocks()[0]
 }
 
 func (ts *f3TipSet) Beacon() []byte {
-	entries := ts.cast().Blocks()[0].BeaconEntries
-	if len(entries) == 0 {
-		// This should never happen in practice, but set beacon to a non-nil
-		// 32byte slice to force the message builder to generate a
-		// ticket. Otherwise, messages that require ticket, i.e. CONVERGE will fail
-		// validation due to the absence of ticket. This is a convoluted way of doing it.
+	switch header := ts.FirstBlockHeader(); {
+	case header == nil, len(header.BeaconEntries) == 0:
+		// This should never happen in practice, but set beacon to a non-nil 32byte slice
+		// to force the message builder to generate a ticket. Otherwise, messages that
+		// require ticket, i.e. CONVERGE will fail validation due to the absence of
+		// ticket. This is a convoluted way of doing it.
+
+		// TODO: investigate if this is still necessary, or how message builder can be
+		//       adapted to behave correctly regardless of beacon value, e.g. fail fast
+		//       instead of building CONVERGE with empty beacon.
 		return make([]byte, 32)
+	default:
+		return header.BeaconEntries[len(header.BeaconEntries)-1].Data
 	}
-	return entries[len(entries)-1].Data
-}
-
-func (ts *f3TipSet) Epoch() int64 {
-	return int64(ts.cast().Height())
 }
 
 func (ts *f3TipSet) Timestamp() time.Time {
-	return time.Unix(int64(ts.cast().Blocks()[0].Timestamp), 0)
-}
-
-func wrapTS(ts *types.TipSet) ec.TipSet {
-	if ts == nil {
-		return nil
+	if header := ts.FirstBlockHeader(); header != nil {
+		return time.Unix(int64(header.Timestamp), 0)
 	}
-	return (*f3TipSet)(ts)
+	return time.Time{}
 }
 
 // GetTipsetByEpoch should return a tipset before the one requested if the requested
@@ -75,57 +81,42 @@ func (ec *ecWrapper) GetTipsetByEpoch(ctx context.Context, epoch int64) (ec.TipS
 	if err != nil {
 		return nil, xerrors.Errorf("getting tipset by height: %w", err)
 	}
-	return wrapTS(ts), nil
+	return &f3TipSet{TipSet: ts}, nil
 }
 
 func (ec *ecWrapper) GetTipset(ctx context.Context, tsk gpbft.TipSetKey) (ec.TipSet, error) {
-	tskLotus, err := types.TipSetKeyFromBytes(tsk)
-	if err != nil {
-		return nil, xerrors.Errorf("decoding tsk: %w", err)
-	}
-
-	ts, err := ec.ChainStore.GetTipSetFromKey(ctx, tskLotus)
+	ts, err := ec.getTipSetFromF3TSK(ctx, tsk)
 	if err != nil {
 		return nil, xerrors.Errorf("getting tipset by key: %w", err)
 	}
 
-	return wrapTS(ts), nil
+	return &f3TipSet{TipSet: ts}, nil
 }
 
-func (ec *ecWrapper) GetHead(_ context.Context) (ec.TipSet, error) {
-	return wrapTS(ec.ChainStore.GetHeaviestTipSet()), nil
+func (ec *ecWrapper) GetHead(context.Context) (ec.TipSet, error) {
+	head := ec.ChainStore.GetHeaviestTipSet()
+	if head == nil {
+		return nil, xerrors.New("no heaviest tipset")
+	}
+	return &f3TipSet{TipSet: head}, nil
 }
 
 func (ec *ecWrapper) GetParent(ctx context.Context, tsF3 ec.TipSet) (ec.TipSet, error) {
-	var ts *types.TipSet
-	if tsW, ok := tsF3.(*f3TipSet); ok {
-		ts = tsW.cast()
-	} else {
-		// There are only two implementations of ec.TipSet: f3TipSet, and one in fake EC
-		// backend.
-		//
-		// TODO: Revisit the type check here and remove as needed once testing
-		//       is over and fake EC backend goes away.
-		tskLotus, err := types.TipSetKeyFromBytes(tsF3.Key())
-		if err != nil {
-			return nil, xerrors.Errorf("decoding tsk: %w", err)
-		}
-		ts, err = ec.ChainStore.GetTipSetFromKey(ctx, tskLotus)
-		if err != nil {
-			return nil, xerrors.Errorf("getting tipset by key for get parent: %w", err)
-		}
+	ts, err := ec.toLotusTipSet(ctx, tsF3)
+	if err != nil {
+		return nil, err
 	}
 	parentTs, err := ec.ChainStore.GetTipSetFromKey(ctx, ts.Parents())
 	if err != nil {
 		return nil, xerrors.Errorf("getting parent tipset: %w", err)
 	}
-	return wrapTS(parentTs), nil
+	return &f3TipSet{TipSet: parentTs}, nil
 }
 
 func (ec *ecWrapper) GetPowerTable(ctx context.Context, tskF3 gpbft.TipSetKey) (gpbft.PowerEntries, error) {
-	tsk, err := types.TipSetKeyFromBytes(tskF3)
+	tsk, err := toLotusTipSetKey(tskF3)
 	if err != nil {
-		return nil, xerrors.Errorf("decoding tsk: %w", err)
+		return nil, err
 	}
 	return ec.getPowerTableLotusTSK(ctx, tsk)
 }
@@ -208,7 +199,7 @@ func (ec *ecWrapper) getPowerTableLotusTSK(ctx context.Context, tsk types.TipSet
 		if waddr.Protocol() != address.BLS {
 			return xerrors.Errorf("wrong type of worker address")
 		}
-		pe.PubKey = gpbft.PubKey(waddr.Payload())
+		pe.PubKey = waddr.Payload()
 		powerEntries = append(powerEntries, pe)
 		return nil
 	})
@@ -219,3 +210,43 @@ func (ec *ecWrapper) getPowerTableLotusTSK(ctx context.Context, tsk types.TipSet
 	sort.Sort(powerEntries)
 	return powerEntries, nil
 }
+
+func (ec *ecWrapper) Finalize(ctx context.Context, key gpbft.TipSetKey) error {
+	if !ec.Checkpoint {
+		return nil // Nothing to do; checkpointing is not enabled.
+	}
+	tsk, err := toLotusTipSetKey(key)
+	if err != nil {
+		return err
+	}
+	if err = ec.Syncer.SyncCheckpoint(ctx, tsk); err != nil {
+		return xerrors.Errorf("checkpointing finalized tipset: %w", err)
+	}
+	return nil
+}
+
+func (ec *ecWrapper) toLotusTipSet(ctx context.Context, ts ec.TipSet) (*types.TipSet, error) {
+	switch tst := ts.(type) {
+	case *f3TipSet:
+		return tst.TipSet, nil
+	default:
+		// Fall back on getting the tipset by key. This path is executed only in testing.
+		return ec.getTipSetFromF3TSK(ctx, ts.Key())
+	}
+}
+
+func (ec *ecWrapper) getTipSetFromF3TSK(ctx context.Context, key gpbft.TipSetKey) (*types.TipSet, error) {
+	tsk, err := toLotusTipSetKey(key)
+	if err != nil {
+		return nil, err
+	}
+	ts, err := ec.ChainStore.GetTipSetFromKey(ctx, tsk)
+	if err != nil {
+		return nil, xerrors.Errorf("getting tipset from key: %w", err)
+	}
+	return ts, nil
+}
+
+func toLotusTipSetKey(key gpbft.TipSetKey) (types.TipSetKey, error) {
+	return types.TipSetKeyFromBytes(key)
+}

chain/lf3/f3.go

@@ -20,6 +20,7 @@ import (
 	"github.com/filecoin-project/go-f3/manifest"
 
 	"github.com/filecoin-project/lotus/api"
+	"github.com/filecoin-project/lotus/chain"
 	"github.com/filecoin-project/lotus/chain/stmgr"
 	"github.com/filecoin-project/lotus/chain/store"
 	"github.com/filecoin-project/lotus/chain/types"
@@ -35,17 +36,21 @@ type F3 struct {
 	newLeases chan leaseRequest
 }
 
+type F3ConsensusEnabled bool
+
 type F3Params struct {
 	fx.In
 
-	NetworkName      dtypes.NetworkName
-	ManifestProvider manifest.ManifestProvider
-	PubSub           *pubsub.PubSub
-	Host             host.Host
-	ChainStore       *store.ChainStore
-	StateManager     *stmgr.StateManager
-	Datastore        dtypes.MetadataDS
-	Wallet           api.Wallet
+	NetworkName        dtypes.NetworkName
+	ManifestProvider   manifest.ManifestProvider
+	PubSub             *pubsub.PubSub
+	Host               host.Host
+	ChainStore         *store.ChainStore
+	Syncer             *chain.Syncer
+	StateManager       *stmgr.StateManager
+	Datastore          dtypes.MetadataDS
+	Wallet             api.Wallet
+	F3ConsensusEnabled F3ConsensusEnabled
 }
 
 var log = logging.Logger("f3")
@@ -56,6 +61,8 @@ func New(mctx helpers.MetricsCtx, lc fx.Lifecycle, params F3Params) (*F3, error)
 	ec := &ecWrapper{
 		ChainStore:   params.ChainStore,
 		StateManager: params.StateManager,
+		Syncer:       params.Syncer,
+		Checkpoint:   bool(params.F3ConsensusEnabled),
 	}
 	verif := blssig.VerifierWithKeyOnG1()
 

chain/types/tipset_key.go

@@ -10,6 +10,7 @@ import (
 	block "github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-cid"
 	typegen "github.com/whyrusleeping/cbor-gen"
+	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
 )
@@ -52,7 +53,7 @@ func NewTipSetKey(cids ...cid.Cid) TipSetKey {
 func TipSetKeyFromBytes(encoded []byte) (TipSetKey, error) {
 	_, err := decodeKey(encoded)
 	if err != nil {
-		return EmptyTSK, err
+		return EmptyTSK, xerrors.Errorf("decoding tpiset key: %w", err)
 	}
 	return TipSetKey{string(encoded)}, nil
 }