Skip to content

Commit

Permalink
eth/gasprice: feeHistory improvements (#23422)
Browse files Browse the repository at this point in the history 
* eth/gasprice: cache feeHistory results

* eth/gasprice: changed feeHistory block count limitation

* eth/gasprice: do not use embedded struct in blockFees

* eth/gasprice: fee processing logic cleanup

* eth/gasprice: purge feeHistory cache at chain reorgs
  • Loading branch information
@zsfelfoldi
zsfelfoldi authored Aug 23, 2021
1 parent dfeb2f7 commit f38abc5
Show file tree
Hide file tree
Showing 5 changed files with 95 additions and 61 deletions.
4 changes: 2 additions & 2 deletions eth/ethconfig/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@ import (
var FullNodeGPO = gasprice.Config{
Blocks: 20,
Percentile: 60,
MaxHeaderHistory: 0,
MaxBlockHistory: 0,
MaxHeaderHistory: 1024,
MaxBlockHistory: 1024,
MaxPrice: gasprice.DefaultMaxPrice,
IgnorePrice: gasprice.DefaultIgnorePrice,
}
Expand Down
105 changes: 57 additions & 48 deletions eth/gasprice/feehistory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ package gasprice

import (
"context"
"encoding/binary"
"errors"
"fmt"
"math"
"math/big"
"sort"
"sync/atomic"
Expand All @@ -37,10 +39,6 @@ var (
)

const (
// maxFeeHistory is the maximum number of blocks that can be retrieved for a
// fee history request.
maxFeeHistory = 1024

// maxBlockFetchers is the max number of goroutines to spin up to pull blocks
// for the fee history calculation (mostly relevant for LES).
maxBlockFetchers = 4
Expand All @@ -54,10 +52,15 @@ type blockFees struct {
block *types.Block // only set if reward percentiles are requested
receipts types.Receipts
// filled by processBlock
results processedFees
err error
}

// processedFees contains the results of a processed block and is also used for caching
type processedFees struct {
reward []*big.Int
baseFee, nextBaseFee *big.Int
gasUsedRatio float64
err error
}

// txGasAndReward is sorted in ascending order based on reward
Expand All @@ -82,15 +85,15 @@ func (s sortGasAndReward) Less(i, j int) bool {
// fills in the rest of the fields.
func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) {
chainconfig := oracle.backend.ChainConfig()
if bf.baseFee = bf.header.BaseFee; bf.baseFee == nil {
bf.baseFee = new(big.Int)
if bf.results.baseFee = bf.header.BaseFee; bf.results.baseFee == nil {
bf.results.baseFee = new(big.Int)
}
if chainconfig.IsLondon(big.NewInt(int64(bf.blockNumber + 1))) {
bf.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header)
bf.results.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header)
} else {
bf.nextBaseFee = new(big.Int)
bf.results.nextBaseFee = new(big.Int)
}
bf.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit)
bf.results.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit)
if len(percentiles) == 0 {
// rewards were not requested, return null
return
Expand All @@ -100,11 +103,11 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) {
return
}

bf.reward = make([]*big.Int, len(percentiles))
bf.results.reward = make([]*big.Int, len(percentiles))
if len(bf.block.Transactions()) == 0 {
// return an all zero row if there are no transactions to gather data from
for i := range bf.reward {
bf.reward[i] = new(big.Int)
for i := range bf.results.reward {
bf.results.reward[i] = new(big.Int)
}
return
}
Expand All @@ -125,7 +128,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) {
txIndex++
sumGasUsed += sorter[txIndex].gasUsed
}
bf.reward[i] = sorter[txIndex].reward
bf.results.reward[i] = sorter[txIndex].reward
}
}

Expand All @@ -134,7 +137,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) {
// also returned if requested and available.
// Note: an error is only returned if retrieving the head header has failed. If there are no
// retrievable blocks in the specified range then zero block count is returned with no error.
func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks, maxHistory int) (*types.Block, []*types.Receipt, uint64, int, error) {
func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) {
var (
headBlock rpc.BlockNumber
pendingBlock *types.Block
Expand Down Expand Up @@ -167,17 +170,6 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.Block
} else if pendingBlock == nil && lastBlock > headBlock {
return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock)
}
if maxHistory != 0 {
// limit retrieval to the given number of latest blocks
if tooOldCount := int64(headBlock) - int64(maxHistory) - int64(lastBlock) + int64(blocks); tooOldCount > 0 {
// tooOldCount is the number of requested blocks that are too old to be served
if int64(blocks) > tooOldCount {
blocks -= int(tooOldCount)
} else {
return nil, nil, 0, 0, nil
}
}
}
// ensure not trying to retrieve before genesis
if rpc.BlockNumber(blocks) > lastBlock+1 {
blocks = int(lastBlock + 1)
Expand All @@ -202,6 +194,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
if blocks < 1 {
return common.Big0, nil, nil, nil, nil // returning with no data and no error means there are no retrievable blocks
}
maxFeeHistory := oracle.maxHeaderHistory
if len(rewardPercentiles) != 0 {
maxFeeHistory = oracle.maxBlockHistory
}
if blocks > maxFeeHistory {
log.Warn("Sanitizing fee history length", "requested", blocks, "truncated", maxFeeHistory)
blocks = maxFeeHistory
Expand All @@ -214,17 +210,12 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p)
}
}
// Only process blocks if reward percentiles were requested
maxHistory := oracle.maxHeaderHistory
if len(rewardPercentiles) != 0 {
maxHistory = oracle.maxBlockHistory
}
var (
pendingBlock *types.Block
pendingReceipts []*types.Receipt
err error
)
pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks, maxHistory)
pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks)
if err != nil || blocks == 0 {
return common.Big0, nil, nil, nil, err
}
Expand All @@ -234,6 +225,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
next = oldestBlock
results = make(chan *blockFees, blocks)
)
percentileKey := make([]byte, 8*len(rewardPercentiles))
for i, p := range rewardPercentiles {
binary.LittleEndian.PutUint64(percentileKey[i*8:(i+1)*8], math.Float64bits(p))
}
for i := 0; i < maxBlockFetchers && i < blocks; i++ {
go func() {
for {
Expand All @@ -246,24 +241,38 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
fees := &blockFees{blockNumber: blockNumber}
if pendingBlock != nil && blockNumber >= pendingBlock.NumberU64() {
fees.block, fees.receipts = pendingBlock, pendingReceipts
fees.header = fees.block.Header()
oracle.processBlock(fees, rewardPercentiles)
results <- fees
} else {
if len(rewardPercentiles) != 0 {
fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber))
if fees.block != nil && fees.err == nil {
fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash())
}
cacheKey := struct {
number uint64
percentiles string
}{blockNumber, string(percentileKey)}

if p, ok := oracle.historyCache.Get(cacheKey); ok {
fees.results = p.(processedFees)
results <- fees
} else {
fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber))
if len(rewardPercentiles) != 0 {
fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber))
if fees.block != nil && fees.err == nil {
fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash())
fees.header = fees.block.Header()
}
} else {
fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber))
}
if fees.header != nil && fees.err == nil {
oracle.processBlock(fees, rewardPercentiles)
if fees.err == nil {
oracle.historyCache.Add(cacheKey, fees.results)
}
}
// send to results even if empty to guarantee that blocks items are sent in total
results <- fees
}
}
if fees.block != nil {
fees.header = fees.block.Header()
}
if fees.header != nil {
oracle.processBlock(fees, rewardPercentiles)
}
// send to results even if empty to guarantee that blocks items are sent in total
results <- fees
}
}()
}
Expand All @@ -279,8 +288,8 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast
return common.Big0, nil, nil, nil, fees.err
}
i := int(fees.blockNumber - oldestBlock)
if fees.header != nil {
reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.reward, fees.baseFee, fees.nextBaseFee, fees.gasUsedRatio
if fees.results.baseFee != nil {
reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.results.reward, fees.results.baseFee, fees.results.nextBaseFee, fees.results.gasUsedRatio
} else {
// getting no block and no error means we are requesting into the future (might happen because of a reorg)
if i < firstMissing {
Expand Down
22 changes: 11 additions & 11 deletions eth/gasprice/feehistory_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,20 +36,20 @@ func TestFeeHistory(t *testing.T) {
expCount int
expErr error
}{
{false, 0, 0, 10, 30, nil, 21, 10, nil},
{false, 0, 0, 10, 30, []float64{0, 10}, 21, 10, nil},
{false, 0, 0, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile},
{false, 0, 0, 1000000000, 30, nil, 0, 31, nil},
{false, 0, 0, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil},
{false, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead},
{true, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead},
{false, 1000, 1000, 10, 30, nil, 21, 10, nil},
{false, 1000, 1000, 10, 30, []float64{0, 10}, 21, 10, nil},
{false, 1000, 1000, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile},
{false, 1000, 1000, 1000000000, 30, nil, 0, 31, nil},
{false, 1000, 1000, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil},
{false, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead},
{true, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead},
{false, 20, 2, 100, rpc.LatestBlockNumber, nil, 13, 20, nil},
{false, 20, 2, 100, rpc.LatestBlockNumber, []float64{0, 10}, 31, 2, nil},
{false, 20, 2, 100, 32, []float64{0, 10}, 31, 2, nil},
{false, 0, 0, 1, rpc.PendingBlockNumber, nil, 0, 0, nil},
{false, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 1, nil},
{true, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 2, nil},
{true, 0, 0, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil},
{false, 1000, 1000, 1, rpc.PendingBlockNumber, nil, 0, 0, nil},
{false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil},
{true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil},
{true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil},
}
for i, c := range cases {
config := Config{
Expand Down
20 changes: 20 additions & 0 deletions eth/gasprice/gasprice.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,13 @@ import (
"sync"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
lru "github.com/hashicorp/golang-lru"
)

const sampleNumber = 3 // Number of transactions sampled in a block
Expand All @@ -53,6 +56,7 @@ type OracleBackend interface {
GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error)
PendingBlockAndReceipts() (*types.Block, types.Receipts)
ChainConfig() *params.ChainConfig
SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
}

// Oracle recommends gas prices based on the content of recent
Expand All @@ -68,6 +72,7 @@ type Oracle struct {

checkBlocks, percentile int
maxHeaderHistory, maxBlockHistory int
historyCache *lru.Cache
}

// NewOracle returns a new gasprice oracle which can recommend suitable
Expand Down Expand Up @@ -99,6 +104,20 @@ func NewOracle(backend OracleBackend, params Config) *Oracle {
} else if ignorePrice.Int64() > 0 {
log.Info("Gasprice oracle is ignoring threshold set", "threshold", ignorePrice)
}

cache, _ := lru.New(2048)
headEvent := make(chan core.ChainHeadEvent, 1)
backend.SubscribeChainHeadEvent(headEvent)
go func() {
var lastHead common.Hash
for ev := range headEvent {
if ev.Block.ParentHash() != lastHead {
cache.Purge()
}
lastHead = ev.Block.Hash()
}
}()

return &Oracle{
backend: backend,
lastPrice: params.Default,
Expand All @@ -108,6 +127,7 @@ func NewOracle(backend OracleBackend, params Config) *Oracle {
percentile: percent,
maxHeaderHistory: params.MaxHeaderHistory,
maxBlockHistory: params.MaxBlockHistory,
historyCache: cache,
}
}

Expand Down
5 changes: 5 additions & 0 deletions eth/gasprice/gasprice_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
)
Expand Down Expand Up @@ -90,6 +91,10 @@ func (b *testBackend) ChainConfig() *params.ChainConfig {
return b.chain.Config()
}

func (b *testBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription {
return nil
}

func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBackend {
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
Expand Down

0 comments on commit f38abc5

Please sign in to comment.