Merge pull request #9654 from filecoin-project/gstuart/gas-estimation…

…-tooling

feat: cli: gas estimation tooling

build/drand.go

@@ -69,6 +69,10 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{
 		ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`,
 	},
 	DrandIncentinet: {
+		Servers: []string{
+			"https://dev1.drand.sh",
+			"https://dev2.drand.sh",
+		},
 		ChainInfoJSON: `{"public_key":"8cad0c72c606ab27d36ee06de1d5b2db1faf92e447025ca37575ab3a8aac2eaae83192f846fc9e158bc738423753d000","period":30,"genesis_time":1595873820,"hash":"80c8b872c714f4c00fdd3daa465d5514049f457f01f85a4caf68cdcd394ba039","groupHash":"d9406aaed487f7af71851b4399448e311f2328923d454e971536c05398ce2d9b"}`,
 	},
 }

chain/beacon/drand/drand.go

@@ -107,7 +107,7 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config dtypes
 
 	client, err := dclient.Wrap(clients, opts...)
 	if err != nil {
-		return nil, xerrors.Errorf("creating drand client")
+		return nil, xerrors.Errorf("creating drand client: %w", err)
 	}
 
 	lc, err := lru.New(1024)

chain/stmgr/call.go

@@ -14,6 +14,7 @@ import (
 	"github.com/filecoin-project/go-state-types/abi"
 	"github.com/filecoin-project/go-state-types/big"
 	"github.com/filecoin-project/go-state-types/crypto"
+	"github.com/filecoin-project/go-state-types/network"
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/blockstore"
@@ -29,74 +30,7 @@ var ErrExpensiveFork = errors.New("refusing explicit call due to state fork at e
 // Call applies the given message to the given tipset's parent state, at the epoch following the
 // tipset's parent. In the presence of null blocks, the height at which the message is invoked may
 // be less than the specified tipset.
-//
-//   - If no tipset is specified, the first tipset without an expensive migration is used.
-//   - If executing a message at a given tipset would trigger an expensive migration, the call will
-//     fail with ErrExpensiveFork.
 func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) {
-	ctx, span := trace.StartSpan(ctx, "statemanager.Call")
-	defer span.End()
-
-	var pheight abi.ChainEpoch = -1
-
-	// If no tipset is provided, try to find one without a fork.
-	if ts == nil {
-		ts = sm.cs.GetHeaviestTipSet()
-		// Search back till we find a height with no fork, or we reach the beginning.
-		for ts.Height() > 0 {
-			pts, err := sm.cs.GetTipSetFromKey(ctx, ts.Parents())
-			if err != nil {
-				return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err)
-			}
-			if !sm.hasExpensiveFork(pts.Height()) {
-				pheight = pts.Height()
-				break
-			}
-			ts = pts
-		}
-	} else if ts.Height() > 0 {
-		pts, err := sm.cs.LoadTipSet(ctx, ts.Parents())
-		if err != nil {
-			return nil, xerrors.Errorf("failed to load parent tipset: %w", err)
-		}
-		pheight = pts.Height()
-		if sm.hasExpensiveFork(pheight) {
-			return nil, ErrExpensiveFork
-		}
-	} else {
-		// We can't get the parent tipset in this case.
-		pheight = ts.Height() - 1
-	}
-
-	// Since we're simulating a future message, pretend we're applying it in the "next" tipset
-	vmHeight := pheight + 1
-	bstate := ts.ParentState()
-
-	// Run the (not expensive) migration.
-	bstate, err := sm.HandleStateForks(ctx, bstate, pheight, nil, ts)
-	if err != nil {
-		return nil, fmt.Errorf("failed to handle fork: %w", err)
-	}
-
-	vmopt := &vm.VMOpts{
-		StateBase:      bstate,
-		Epoch:          vmHeight,
-		Rand:           rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion),
-		Bstore:         sm.cs.StateBlockstore(),
-		Actors:         sm.tsExec.NewActorRegistry(),
-		Syscalls:       sm.Syscalls,
-		CircSupplyCalc: sm.GetVMCirculatingSupply,
-		NetworkVersion: sm.GetNetworkVersion(ctx, pheight+1),
-		BaseFee:        types.NewInt(0),
-		LookbackState:  LookbackStateGetterForTipset(sm, ts),
-		Tracing:        true,
-	}
-
-	vmi, err := sm.newVM(ctx, vmopt)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to set up vm: %w", err)
-	}
-
 	if msg.GasLimit == 0 {
 		msg.GasLimit = build.BlockGasLimit
 	}
@@ -106,61 +40,43 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
 	if msg.GasPremium == types.EmptyInt {
 		msg.GasPremium = types.NewInt(0)
 	}
-
 	if msg.Value == types.EmptyInt {
 		msg.Value = types.NewInt(0)
 	}
 
-	if span.IsRecordingEvents() {
-		span.AddAttributes(
-			trace.Int64Attribute("gas_limit", msg.GasLimit),
-			trace.StringAttribute("gas_feecap", msg.GasFeeCap.String()),
-			trace.StringAttribute("value", msg.Value.String()),
-		)
-	}
-
-	stTree, err := sm.StateTree(bstate)
-	if err != nil {
-		return nil, xerrors.Errorf("failed to load state tree: %w", err)
-	}
-
-	fromActor, err := stTree.GetActor(msg.From)
-	if err != nil {
-		return nil, xerrors.Errorf("call raw get actor: %s", err)
-	}
-
-	msg.Nonce = fromActor.Nonce
+	return sm.callInternal(ctx, msg, nil, ts, cid.Undef, sm.GetNetworkVersion, false)
+}
 
-	// TODO: maybe just use the invoker directly?
-	ret, err := vmi.ApplyImplicitMessage(ctx, msg)
-	if err != nil && ret == nil {
-		return nil, xerrors.Errorf("apply message failed: %w", err)
-	}
+// CallWithGas calculates the state for a given tipset, and then applies the given message on top of that state.
+func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet) (*api.InvocResult, error) {
+	return sm.callInternal(ctx, msg, priorMsgs, ts, cid.Undef, sm.GetNetworkVersion, true)
+}
 
-	var errs string
-	if ret.ActorErr != nil {
-		errs = ret.ActorErr.Error()
-		log.Warnf("chain call failed: %s", ret.ActorErr)
+// CallAtStateAndVersion allows you to specify a message to execute on the given stateCid and network version.
+// This should mostly be used for gas modelling on a migrated state.
+// Tipset here is not needed because stateCid and network version fully describe execution we want. The internal function
+// will get the heaviest tipset for use for things like basefee, which we don't really care about here.
+func (sm *StateManager) CallAtStateAndVersion(ctx context.Context, msg *types.Message, stateCid cid.Cid, v network.Version) (*api.InvocResult, error) {
+	nvGetter := func(context.Context, abi.ChainEpoch) network.Version {
+		return v
 	}
 
-	return &api.InvocResult{
-		MsgCid:         msg.Cid(),
-		Msg:            msg,
-		MsgRct:         &ret.MessageReceipt,
-		ExecutionTrace: ret.ExecutionTrace,
-		Error:          errs,
-		Duration:       ret.Duration,
-	}, err
+	return sm.callInternal(ctx, msg, nil, nil, stateCid, nvGetter, true)
 }
 
-func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet) (*api.InvocResult, error) {
-	ctx, span := trace.StartSpan(ctx, "statemanager.CallWithGas")
+//   - If no tipset is specified, the first tipset without an expensive migration or one in its parent is used.
+//   - If executing a message at a given tipset or its parent would trigger an expensive migration, the call will
+//     fail with ErrExpensiveFork.
+func (sm *StateManager) callInternal(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet, stateCid cid.Cid, nvGetter rand.NetworkVersionGetter, checkGas bool) (*api.InvocResult, error) {
+	ctx, span := trace.StartSpan(ctx, "statemanager.callInternal")
 	defer span.End()
 
 	// Copy the message as we'll be modifying the nonce.
 	msgCopy := *msg
 	msg = &msgCopy
 
+	var err error
+	var pts *types.TipSet
 	if ts == nil {
 		ts = sm.cs.GetHeaviestTipSet()
 
@@ -170,18 +86,19 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		// height to have no fork, because we'll run it inside this
 		// function before executing the given message.
 		for ts.Height() > 0 {
-			pts, err := sm.cs.GetTipSetFromKey(ctx, ts.Parents())
+			pts, err = sm.cs.GetTipSetFromKey(ctx, ts.Parents())
 			if err != nil {
 				return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err)
 			}
+			// Checks for expensive forks from the parents to the tipset, including nil tipsets
 			if !sm.hasExpensiveForkBetween(pts.Height(), ts.Height()+1) {
 				break
 			}
 
 			ts = pts
 		}
 	} else if ts.Height() > 0 {
-		pts, err := sm.cs.GetTipSetFromKey(ctx, ts.Parents())
+		pts, err = sm.cs.GetTipSetFromKey(ctx, ts.Parents())
 		if err != nil {
 			return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err)
 		}
@@ -190,12 +107,22 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		}
 	}
 
-	// Since we're simulating a future message, pretend we're applying it in the "next" tipset
-	vmHeight := ts.Height() + 1
-
-	stateCid, _, err := sm.TipSetState(ctx, ts)
-	if err != nil {
-		return nil, xerrors.Errorf("computing tipset state: %w", err)
+	var vmHeight abi.ChainEpoch
+	if checkGas {
+		// Since we're simulating a future message, pretend we're applying it in the "next" tipset
+		vmHeight = ts.Height() + 1
+		if stateCid == cid.Undef {
+			stateCid, _, err = sm.TipSetState(ctx, ts)
+			if err != nil {
+				return nil, xerrors.Errorf("computing tipset state: %w", err)
+			}
+		}
+	} else {
+		// If we're not checking gas, we don't want to have to execute the tipset like above. This saves a lot of computation time
+		vmHeight = pts.Height() + 1
+		if stateCid == cid.Undef {
+			stateCid = ts.ParentState()
+		}
 	}
 
 	// Technically, the tipset we're passing in here should be ts+1, but that may not exist.
@@ -204,8 +131,6 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		return nil, fmt.Errorf("failed to handle fork: %w", err)
 	}
 
-	r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion)
-
 	if span.IsRecordingEvents() {
 		span.AddAttributes(
 			trace.Int64Attribute("gas_limit", msg.GasLimit),
@@ -218,12 +143,12 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 	vmopt := &vm.VMOpts{
 		StateBase:      stateCid,
 		Epoch:          vmHeight,
-		Rand:           r,
+		Rand:           rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, nvGetter),
 		Bstore:         buffStore,
 		Actors:         sm.tsExec.NewActorRegistry(),
 		Syscalls:       sm.Syscalls,
 		CircSupplyCalc: sm.GetVMCirculatingSupply,
-		NetworkVersion: sm.GetNetworkVersion(ctx, ts.Height()+1),
+		NetworkVersion: nvGetter(ctx, vmHeight),
 		BaseFee:        ts.Blocks()[0].ParentBaseFee,
 		LookbackState:  LookbackStateGetterForTipset(sm, ts),
 		Tracing:        true,
@@ -233,7 +158,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		return nil, xerrors.Errorf("failed to set up vm: %w", err)
 	}
 	for i, m := range priorMsgs {
-		_, err := vmi.ApplyMessage(ctx, m)
+		_, err = vmi.ApplyMessage(ctx, m)
 		if err != nil {
 			return nil, xerrors.Errorf("applying prior message (%d, %s): %w", i, m.Cid(), err)
 		}
@@ -258,27 +183,6 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 
 	msg.Nonce = fromActor.Nonce
 
-	fromKey, err := sm.ResolveToKeyAddress(ctx, msg.From, ts)
-	if err != nil {
-		return nil, xerrors.Errorf("could not resolve key: %w", err)
-	}
-
-	var msgApply types.ChainMsg
-
-	switch fromKey.Protocol() {
-	case address.BLS:
-		msgApply = msg
-	case address.SECP256K1:
-		msgApply = &types.SignedMessage{
-			Message: *msg,
-			Signature: crypto.Signature{
-				Type: crypto.SigTypeSecp256k1,
-				Data: make([]byte, 65),
-			},
-		}
-
-	}
-
 	// If the fee cap is set to zero, make gas free.
 	if msg.GasFeeCap.NilOrZero() {
 		// Now estimate with a new VM with no base fee.
@@ -291,9 +195,39 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		}
 	}
 
-	ret, err := vmi.ApplyMessage(ctx, msgApply)
-	if err != nil {
-		return nil, xerrors.Errorf("gas estimation failed: %w", err)
+	var ret *vm.ApplyRet
+	var gasInfo api.MsgGasCost
+	if checkGas {
+		fromKey, err := sm.ResolveToKeyAddress(ctx, msg.From, ts)
+		if err != nil {
+			return nil, xerrors.Errorf("could not resolve key: %w", err)
+		}
+
+		var msgApply types.ChainMsg
+
+		switch fromKey.Protocol() {
+		case address.BLS:
+			msgApply = msg
+		case address.SECP256K1:
+			msgApply = &types.SignedMessage{
+				Message: *msg,
+				Signature: crypto.Signature{
+					Type: crypto.SigTypeSecp256k1,
+					Data: make([]byte, 65),
+				},
+			}
+		}
+
+		ret, err = vmi.ApplyMessage(ctx, msgApply)
+		if err != nil {
+			return nil, xerrors.Errorf("gas estimation failed: %w", err)
+		}
+		gasInfo = MakeMsgGasCost(msg, ret)
+	} else {
+		ret, err = vmi.ApplyImplicitMessage(ctx, msg)
+		if err != nil && ret == nil {
+			return nil, xerrors.Errorf("apply message failed: %w", err)
+		}
 	}
 
 	var errs string
@@ -305,11 +239,11 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
 		MsgCid:         msg.Cid(),
 		Msg:            msg,
 		MsgRct:         &ret.MessageReceipt,
-		GasCost:        MakeMsgGasCost(msg, ret),
+		GasCost:        gasInfo,
 		ExecutionTrace: ret.ExecutionTrace,
 		Error:          errs,
 		Duration:       ret.Duration,
-	}, nil
+	}, err
 }
 
 var errHaltExecution = fmt.Errorf("halt")

chain/stmgr/forks_test.go

@@ -335,7 +335,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
 		parentHeight := pts.Height()
 		currentHeight := ts.TipSet.TipSet().Height()
 
-		// CallWithGas calls _at_ the current tipset.
+		// CallWithGas calls on top of the given tipset.
 		ret, err := sm.CallWithGas(ctx, m, nil, ts.TipSet.TipSet())
 		if parentHeight <= testForkHeight && currentHeight >= testForkHeight {
 			// If I had a fork, or I _will_ have a fork, it should fail.
@@ -347,7 +347,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
 
 		// Call always applies the message to the "next block" after the tipset's parent state.
 		ret, err = sm.Call(ctx, m, ts.TipSet.TipSet())
-		if parentHeight == testForkHeight {
+		if parentHeight <= testForkHeight && currentHeight >= testForkHeight {
 			require.Equal(t, ErrExpensiveFork, err)
 		} else {
 			require.NoError(t, err)

chain/types/execresult.go

@@ -42,6 +42,25 @@ type Loc struct {
 	Function string
 }
 
+func (et ExecutionTrace) SumGas() GasTrace {
+	return SumGas(et.GasCharges)
+}
+
+func SumGas(charges []*GasTrace) GasTrace {
+	var out GasTrace
+	for _, gc := range charges {
+		out.TotalGas += gc.TotalGas
+		out.ComputeGas += gc.ComputeGas
+		out.StorageGas += gc.StorageGas
+
+		out.TotalVirtualGas += gc.TotalVirtualGas
+		out.VirtualComputeGas += gc.VirtualComputeGas
+		out.VirtualStorageGas += gc.VirtualStorageGas
+	}
+
+	return out
+}
+
 func (l Loc) Show() bool {
 	ignorePrefix := []string{
 		"reflect.",