Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: cli: gas estimation tooling #9654

Merged
merged 4 commits into from
Nov 22, 2022
Merged

feat: cli: gas estimation tooling #9654

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cde4b80
gas estimation shed command
geoff-vball Nov 16, 2022
62fedfb
Make pre-migration optional
geoff-vball Nov 14, 2022
21afb3f
Review fixes
geoff-vball Nov 16, 2022
9603500
Fold together call functions
geoff-vball Nov 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions build/drand.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ var DrandConfigs = map[DrandEnum]dtypes.DrandConfig{
ChainInfoJSON: `{"public_key":"8cda589f88914aa728fd183f383980b35789ce81b274e5daee1f338b77d02566ef4d3fb0098af1f844f10f9c803c1827","period":25,"genesis_time":1595348225,"hash":"e73b7dc3c4f6a236378220c0dd6aa110eb16eed26c11259606e07ee122838d4f","groupHash":"567d4785122a5a3e75a9bc9911d7ea807dd85ff76b78dc4ff06b075712898607"}`,
},
DrandIncentinet: {
Servers: []string{
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's going on here?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Due to some weird initialization stuff I do, the old beacon needs that. No idea why, that was the quickest fix for hacky code.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@geoff-vball Can you check if this is needed as part of this PR?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is needed in the sense that if I remove it everything breaks. Whether or not its the best way to do things another question

"https://dev1.drand.sh",
"https://dev2.drand.sh",
},
ChainInfoJSON: `{"public_key":"8cad0c72c606ab27d36ee06de1d5b2db1faf92e447025ca37575ab3a8aac2eaae83192f846fc9e158bc738423753d000","period":30,"genesis_time":1595873820,"hash":"80c8b872c714f4c00fdd3daa465d5514049f457f01f85a4caf68cdcd394ba039","groupHash":"d9406aaed487f7af71851b4399448e311f2328923d454e971536c05398ce2d9b"}`,
},
}
2 changes: 1 addition & 1 deletion chain/beacon/drand/drand.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func NewDrandBeacon(genesisTs, interval uint64, ps *pubsub.PubSub, config dtypes

client, err := dclient.Wrap(clients, opts...)
if err != nil {
return nil, xerrors.Errorf("creating drand client")
return nil, xerrors.Errorf("creating drand client: %w", err)
}

lc, err := lru.New(1024)
Expand Down
224 changes: 79 additions & 145 deletions chain/stmgr/call.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"
"github.com/filecoin-project/go-state-types/crypto"
"github.com/filecoin-project/go-state-types/network"

"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/blockstore"
Expand All @@ -29,74 +30,7 @@ var ErrExpensiveFork = errors.New("refusing explicit call due to state fork at e
// Call applies the given message to the given tipset's parent state, at the epoch following the
// tipset's parent. In the presence of null blocks, the height at which the message is invoked may
// be less than the specified tipset.
//
// - If no tipset is specified, the first tipset without an expensive migration is used.
// - If executing a message at a given tipset would trigger an expensive migration, the call will
// fail with ErrExpensiveFork.
func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.TipSet) (*api.InvocResult, error) {
ctx, span := trace.StartSpan(ctx, "statemanager.Call")
defer span.End()

var pheight abi.ChainEpoch = -1

// If no tipset is provided, try to find one without a fork.
if ts == nil {
ts = sm.cs.GetHeaviestTipSet()
// Search back till we find a height with no fork, or we reach the beginning.
for ts.Height() > 0 {
pts, err := sm.cs.GetTipSetFromKey(ctx, ts.Parents())
if err != nil {
return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err)
}
if !sm.hasExpensiveFork(pts.Height()) {
pheight = pts.Height()
break
}
ts = pts
}
} else if ts.Height() > 0 {
pts, err := sm.cs.LoadTipSet(ctx, ts.Parents())
if err != nil {
return nil, xerrors.Errorf("failed to load parent tipset: %w", err)
}
pheight = pts.Height()
if sm.hasExpensiveFork(pheight) {
return nil, ErrExpensiveFork
}
} else {
// We can't get the parent tipset in this case.
pheight = ts.Height() - 1
}

// Since we're simulating a future message, pretend we're applying it in the "next" tipset
vmHeight := pheight + 1
bstate := ts.ParentState()

// Run the (not expensive) migration.
bstate, err := sm.HandleStateForks(ctx, bstate, pheight, nil, ts)
if err != nil {
return nil, fmt.Errorf("failed to handle fork: %w", err)
}

vmopt := &vm.VMOpts{
StateBase: bstate,
Epoch: vmHeight,
Rand: rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion),
Bstore: sm.cs.StateBlockstore(),
Actors: sm.tsExec.NewActorRegistry(),
Syscalls: sm.Syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NetworkVersion: sm.GetNetworkVersion(ctx, pheight+1),
BaseFee: types.NewInt(0),
LookbackState: LookbackStateGetterForTipset(sm, ts),
Tracing: true,
}

vmi, err := sm.newVM(ctx, vmopt)
if err != nil {
return nil, xerrors.Errorf("failed to set up vm: %w", err)
}

if msg.GasLimit == 0 {
msg.GasLimit = build.BlockGasLimit
}
Expand All @@ -106,61 +40,43 @@ func (sm *StateManager) Call(ctx context.Context, msg *types.Message, ts *types.
if msg.GasPremium == types.EmptyInt {
msg.GasPremium = types.NewInt(0)
}

if msg.Value == types.EmptyInt {
msg.Value = types.NewInt(0)
}

if span.IsRecordingEvents() {
span.AddAttributes(
trace.Int64Attribute("gas_limit", msg.GasLimit),
trace.StringAttribute("gas_feecap", msg.GasFeeCap.String()),
trace.StringAttribute("value", msg.Value.String()),
)
}

stTree, err := sm.StateTree(bstate)
if err != nil {
return nil, xerrors.Errorf("failed to load state tree: %w", err)
}

fromActor, err := stTree.GetActor(msg.From)
if err != nil {
return nil, xerrors.Errorf("call raw get actor: %s", err)
}

msg.Nonce = fromActor.Nonce
return sm.callInternal(ctx, msg, nil, ts, cid.Undef, sm.GetNetworkVersion, false)
}

// TODO: maybe just use the invoker directly?
ret, err := vmi.ApplyImplicitMessage(ctx, msg)
if err != nil && ret == nil {
return nil, xerrors.Errorf("apply message failed: %w", err)
}
// CallWithGas calculates the state for a given tipset, and then applies the given message on top of that state.
func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet) (*api.InvocResult, error) {
return sm.callInternal(ctx, msg, priorMsgs, ts, cid.Undef, sm.GetNetworkVersion, true)
}

var errs string
if ret.ActorErr != nil {
errs = ret.ActorErr.Error()
log.Warnf("chain call failed: %s", ret.ActorErr)
// CallAtStateAndVersion allows you to specify a message to execute on the given stateCid and network version.
// This should mostly be used for gas modelling on a migrated state.
// Tipset here is not needed because stateCid and network version fully describe execution we want. The internal function
// will get the heaviest tipset for use for things like basefee, which we don't really care about here.
func (sm *StateManager) CallAtStateAndVersion(ctx context.Context, msg *types.Message, stateCid cid.Cid, v network.Version) (*api.InvocResult, error) {
nvGetter := func(context.Context, abi.ChainEpoch) network.Version {
return v
}

return &api.InvocResult{
MsgCid: msg.Cid(),
Msg: msg,
MsgRct: &ret.MessageReceipt,
ExecutionTrace: ret.ExecutionTrace,
Error: errs,
Duration: ret.Duration,
}, err
return sm.callInternal(ctx, msg, nil, nil, stateCid, nvGetter, true)
}

func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet) (*api.InvocResult, error) {
ctx, span := trace.StartSpan(ctx, "statemanager.CallWithGas")
// - If no tipset is specified, the first tipset without an expensive migration or one in its parent is used.
// - If executing a message at a given tipset or its parent would trigger an expensive migration, the call will
// fail with ErrExpensiveFork.
func (sm *StateManager) callInternal(ctx context.Context, msg *types.Message, priorMsgs []types.ChainMsg, ts *types.TipSet, stateCid cid.Cid, nvGetter rand.NetworkVersionGetter, checkGas bool) (*api.InvocResult, error) {
ctx, span := trace.StartSpan(ctx, "statemanager.callInternal")
defer span.End()

// Copy the message as we'll be modifying the nonce.
msgCopy := *msg
msg = &msgCopy

var err error
var pts *types.TipSet
if ts == nil {
ts = sm.cs.GetHeaviestTipSet()

Expand All @@ -170,18 +86,19 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
// height to have no fork, because we'll run it inside this
// function before executing the given message.
for ts.Height() > 0 {
pts, err := sm.cs.GetTipSetFromKey(ctx, ts.Parents())
pts, err = sm.cs.GetTipSetFromKey(ctx, ts.Parents())
if err != nil {
return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err)
}
// Checks for expensive forks from the parents to the tipset, including nil tipsets
if !sm.hasExpensiveForkBetween(pts.Height(), ts.Height()+1) {
break
}

ts = pts
}
} else if ts.Height() > 0 {
pts, err := sm.cs.GetTipSetFromKey(ctx, ts.Parents())
pts, err = sm.cs.GetTipSetFromKey(ctx, ts.Parents())
if err != nil {
return nil, xerrors.Errorf("failed to find a non-forking epoch: %w", err)
}
Expand All @@ -190,12 +107,22 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
}
}

// Since we're simulating a future message, pretend we're applying it in the "next" tipset
vmHeight := ts.Height() + 1

stateCid, _, err := sm.TipSetState(ctx, ts)
if err != nil {
return nil, xerrors.Errorf("computing tipset state: %w", err)
var vmHeight abi.ChainEpoch
if checkGas {
// Since we're simulating a future message, pretend we're applying it in the "next" tipset
vmHeight = ts.Height() + 1
if stateCid == cid.Undef {
stateCid, _, err = sm.TipSetState(ctx, ts)
if err != nil {
return nil, xerrors.Errorf("computing tipset state: %w", err)
}
}
} else {
// If we're not checking gas, we don't want to have to execute the tipset like above. This saves a lot of computation time
vmHeight = pts.Height() + 1
if stateCid == cid.Undef {
stateCid = ts.ParentState()
}
}

// Technically, the tipset we're passing in here should be ts+1, but that may not exist.
Expand All @@ -204,8 +131,6 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
return nil, fmt.Errorf("failed to handle fork: %w", err)
}

r := rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, sm.GetNetworkVersion)

if span.IsRecordingEvents() {
span.AddAttributes(
trace.Int64Attribute("gas_limit", msg.GasLimit),
Expand All @@ -218,12 +143,12 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
vmopt := &vm.VMOpts{
StateBase: stateCid,
Epoch: vmHeight,
Rand: r,
Rand: rand.NewStateRand(sm.cs, ts.Cids(), sm.beacon, nvGetter),
Bstore: buffStore,
Actors: sm.tsExec.NewActorRegistry(),
Syscalls: sm.Syscalls,
CircSupplyCalc: sm.GetVMCirculatingSupply,
NetworkVersion: sm.GetNetworkVersion(ctx, ts.Height()+1),
NetworkVersion: nvGetter(ctx, vmHeight),
BaseFee: ts.Blocks()[0].ParentBaseFee,
LookbackState: LookbackStateGetterForTipset(sm, ts),
Tracing: true,
Expand All @@ -233,7 +158,7 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
return nil, xerrors.Errorf("failed to set up vm: %w", err)
}
for i, m := range priorMsgs {
_, err := vmi.ApplyMessage(ctx, m)
_, err = vmi.ApplyMessage(ctx, m)
if err != nil {
return nil, xerrors.Errorf("applying prior message (%d, %s): %w", i, m.Cid(), err)
}
Expand All @@ -258,27 +183,6 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri

msg.Nonce = fromActor.Nonce

fromKey, err := sm.ResolveToKeyAddress(ctx, msg.From, ts)
if err != nil {
return nil, xerrors.Errorf("could not resolve key: %w", err)
}

var msgApply types.ChainMsg

switch fromKey.Protocol() {
case address.BLS:
msgApply = msg
case address.SECP256K1:
msgApply = &types.SignedMessage{
Message: *msg,
Signature: crypto.Signature{
Type: crypto.SigTypeSecp256k1,
Data: make([]byte, 65),
},
}

}

// If the fee cap is set to zero, make gas free.
if msg.GasFeeCap.NilOrZero() {
// Now estimate with a new VM with no base fee.
Expand All @@ -291,9 +195,39 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
}
}

ret, err := vmi.ApplyMessage(ctx, msgApply)
if err != nil {
return nil, xerrors.Errorf("gas estimation failed: %w", err)
var ret *vm.ApplyRet
var gasInfo api.MsgGasCost
if checkGas {
fromKey, err := sm.ResolveToKeyAddress(ctx, msg.From, ts)
if err != nil {
return nil, xerrors.Errorf("could not resolve key: %w", err)
}

var msgApply types.ChainMsg
geoff-vball marked this conversation as resolved.
Show resolved Hide resolved

switch fromKey.Protocol() {
case address.BLS:
msgApply = msg
case address.SECP256K1:
msgApply = &types.SignedMessage{
Message: *msg,
Signature: crypto.Signature{
Type: crypto.SigTypeSecp256k1,
Data: make([]byte, 65),
},
}
}

ret, err = vmi.ApplyMessage(ctx, msgApply)
if err != nil {
return nil, xerrors.Errorf("gas estimation failed: %w", err)
}
gasInfo = MakeMsgGasCost(msg, ret)
} else {
ret, err = vmi.ApplyImplicitMessage(ctx, msg)
if err != nil && ret == nil {
return nil, xerrors.Errorf("apply message failed: %w", err)
}
}

var errs string
Expand All @@ -305,11 +239,11 @@ func (sm *StateManager) CallWithGas(ctx context.Context, msg *types.Message, pri
MsgCid: msg.Cid(),
Msg: msg,
MsgRct: &ret.MessageReceipt,
GasCost: MakeMsgGasCost(msg, ret),
GasCost: gasInfo,
ExecutionTrace: ret.ExecutionTrace,
Error: errs,
Duration: ret.Duration,
}, nil
}, err
}

var errHaltExecution = fmt.Errorf("halt")
Expand Down
4 changes: 2 additions & 2 deletions chain/stmgr/forks_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {
parentHeight := pts.Height()
currentHeight := ts.TipSet.TipSet().Height()

// CallWithGas calls _at_ the current tipset.
// CallWithGas calls on top of the given tipset.
ret, err := sm.CallWithGas(ctx, m, nil, ts.TipSet.TipSet())
if parentHeight <= testForkHeight && currentHeight >= testForkHeight {
// If I had a fork, or I _will_ have a fork, it should fail.
Expand All @@ -347,7 +347,7 @@ func testForkRefuseCall(t *testing.T, nullsBefore, nullsAfter int) {

// Call always applies the message to the "next block" after the tipset's parent state.
ret, err = sm.Call(ctx, m, ts.TipSet.TipSet())
if parentHeight == testForkHeight {
if parentHeight <= testForkHeight && currentHeight >= testForkHeight {
require.Equal(t, ErrExpensiveFork, err)
} else {
require.NoError(t, err)
Expand Down
19 changes: 19 additions & 0 deletions chain/types/execresult.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,25 @@ type Loc struct {
Function string
}

func (et ExecutionTrace) SumGas() GasTrace {
return SumGas(et.GasCharges)
}

func SumGas(charges []*GasTrace) GasTrace {
var out GasTrace
for _, gc := range charges {
out.TotalGas += gc.TotalGas
out.ComputeGas += gc.ComputeGas
out.StorageGas += gc.StorageGas

out.TotalVirtualGas += gc.TotalVirtualGas
out.VirtualComputeGas += gc.VirtualComputeGas
out.VirtualStorageGas += gc.VirtualStorageGas
}

return out
}

func (l Loc) Show() bool {
ignorePrefix := []string{
"reflect.",
Expand Down
Loading