From 3c4693d5e7fb1088371e9aec60919a146bc9c484 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 17 Sep 2021 17:35:13 -0400 Subject: [PATCH 1/5] apply the bloom filter when query the ethlogs with a range of blocks --- ethereum/rpc/backend/backend.go | 64 +++++++++ ethereum/rpc/namespaces/eth/filters/api.go | 2 + .../rpc/namespaces/eth/filters/filters.go | 136 +++++++++--------- tests/rpc/rpc_test.go | 38 ++++- 4 files changed, 174 insertions(+), 66 deletions(-) diff --git a/ethereum/rpc/backend/backend.go b/ethereum/rpc/backend/backend.go index a18530ec4c..b6ad57a3ef 100644 --- a/ethereum/rpc/backend/backend.go +++ b/ethereum/rpc/backend/backend.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/tharsis/ethermint/ethereum/rpc/namespaces/eth/filters" "github.com/tharsis/ethermint/ethereum/rpc/types" "github.com/tharsis/ethermint/server/config" ethermint "github.com/tharsis/ethermint/types" @@ -57,6 +58,7 @@ type Backend interface { EstimateGas(args evmtypes.CallArgs, blockNrOptional *types.BlockNumber) (hexutil.Uint64, error) RPCGasCap() uint64 RPCMinGasPrice() int64 + GetFilteredBlocks(from int64, to int64, filter [][]filters.BloomIV, filterAddresses bool) ([]int64, error) } var _ Backend = (*EVMBackend)(nil) @@ -713,3 +715,65 @@ func (e *EVMBackend) RPCMinGasPrice() int64 { return ethermint.DefaultGasPrice } + +// GetFilteredBlocks returns the block height list match the given bloom filters. +func (e *EVMBackend) GetFilteredBlocks( + from int64, + to int64, + filters [][]filters.BloomIV, + filterAddresses bool, +) ([]int64, error) { + matchedBlocks := make([]int64, 0) + +BLOCKS: + for height := from; height <= to; height++ { + if err := e.ctx.Err(); err != nil { + e.logger.Error("EVMBackend context error", "err", err) + return nil, err + } + + h := height + bloom, err := e.BlockBloom(&h) + if err != nil { + e.logger.Error("retrieve header failed", "blockHeight", height, "err", err) + return nil, err + } + + for i, filter := range filters { + // filter the header bloom with the addresses + if filterAddresses && i == 0 { + if !checkMatches(bloom, filter) { + continue BLOCKS + } + + // the filter doesn't have any topics + if len(filters) == 1 { + matchedBlocks = append(matchedBlocks, height) + continue BLOCKS + } + continue + } + + // filter the bloom with topics + if len(filter) > 0 && !checkMatches(bloom, filter) { + continue BLOCKS + } + } + matchedBlocks = append(matchedBlocks, height) + } + + return matchedBlocks, nil +} + +// checkMatches revised the function from +// https://github.com/ethereum/go-ethereum/blob/401354976bb44f0ad4455ca1e0b5c0dc31d9a5f5/core/types/bloom9.go#L88 +func checkMatches(bloom ethtypes.Bloom, filter []filters.BloomIV) bool { + for _, bloomIV := range filter { + if bloomIV.V[0] == bloomIV.V[0]&bloom[bloomIV.I[0]] && + bloomIV.V[1] == bloomIV.V[1]&bloom[bloomIV.I[1]] && + bloomIV.V[2] == bloomIV.V[2]&bloom[bloomIV.I[2]] { + return true + } + } + return false +} diff --git a/ethereum/rpc/namespaces/eth/filters/api.go b/ethereum/rpc/namespaces/eth/filters/api.go index c65dc43ee9..3d40cb946f 100644 --- a/ethereum/rpc/namespaces/eth/filters/api.go +++ b/ethereum/rpc/namespaces/eth/filters/api.go @@ -32,6 +32,8 @@ type Backend interface { GetTransactionLogs(txHash common.Hash) ([]*ethtypes.Log, error) BloomStatus() (uint64, uint64) + + GetFilteredBlocks(from int64, to int64, bloomIndexes [][]BloomIV, filterAddresses bool) ([]int64, error) } // consider a filter inactive if it has not been polled for within deadline diff --git a/ethereum/rpc/namespaces/eth/filters/filters.go b/ethereum/rpc/namespaces/eth/filters/filters.go index 3e7f6a7e61..989b784171 100644 --- a/ethereum/rpc/namespaces/eth/filters/filters.go +++ b/ethereum/rpc/namespaces/eth/filters/filters.go @@ -2,7 +2,7 @@ package filters import ( "context" - "fmt" + "encoding/binary" "math/big" "github.com/tharsis/ethermint/ethereum/rpc/types" @@ -11,17 +11,25 @@ import ( "github.com/tendermint/tendermint/libs/log" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/bloombits" ethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/filters" ) +// BloomIV represents the bit indexes and value inside the bloom filter that belong +// to some key. +type BloomIV struct { + I [3]uint + V [3]byte +} + // Filter can be used to retrieve and filter logs. type Filter struct { logger log.Logger backend Backend criteria filters.FilterCriteria - matcher *bloombits.Matcher + + bloomFilters [][]BloomIV // Filter the system is matching for } // NewBlockFilter creates a new filter which directly inspects the contents of @@ -54,8 +62,6 @@ func NewRangeFilter(logger log.Logger, backend Backend, begin, end int64, addres filtersBz = append(filtersBz, filter) } - size, _ := backend.BloomStatus() - // Create a generic filter and convert it into a range filter criteria := filters.FilterCriteria{ FromBlock: big.NewInt(begin), @@ -64,16 +70,16 @@ func NewRangeFilter(logger log.Logger, backend Backend, begin, end int64, addres Topics: topics, } - return newFilter(logger, backend, criteria, bloombits.NewMatcher(size, filtersBz)) + return newFilter(logger, backend, criteria, createBloomFilters(filtersBz)) } // newFilter returns a new Filter -func newFilter(logger log.Logger, backend Backend, criteria filters.FilterCriteria, matcher *bloombits.Matcher) *Filter { +func newFilter(logger log.Logger, backend Backend, criteria filters.FilterCriteria, bloomFilters [][]BloomIV) *Filter { return &Filter{ - logger: logger, - backend: backend, - criteria: criteria, - matcher: matcher, + logger: logger, + backend: backend, + criteria: criteria, + bloomFilters: bloomFilters, } } @@ -132,52 +138,25 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) { f.criteria.ToBlock = big.NewInt(head + maxToOverhang) } - for i := f.criteria.FromBlock.Int64(); i <= f.criteria.ToBlock.Int64(); i++ { - block, err := f.backend.GetBlockByNumber(types.BlockNumber(i), false) - if err != nil { - return logs, errors.Wrapf(err, "failed to fetch block by number %d", i) - } + from := f.criteria.FromBlock.Int64() + to := f.criteria.ToBlock.Int64() - if block["transactions"] == nil { - continue - } - - var txHashes []common.Hash - - txs, ok := block["transactions"].([]interface{}) - if !ok { - _, ok = block["transactions"].([]common.Hash) - if !ok { - f.logger.Error( - "reading transactions from block data failed", - "type", fmt.Sprintf("%T", block["transactions"]), - ) - } - - continue - } + blocks, err := f.backend.GetFilteredBlocks(from, to, f.bloomFilters, len(f.criteria.Addresses) > 0) + if err != nil { + return nil, err + } - if len(txs) == 0 { - continue + for _, height := range blocks { + ethLogs, err := f.backend.GetLogsByNumber(types.BlockNumber(height)) + if err != nil { + return logs, errors.Wrapf(err, "failed to fetch block by number %d", height) } - for _, tx := range txs { - txHash, ok := tx.(common.Hash) - if !ok { - f.logger.Error( - "transactions list contains non-hash element", - "type", fmt.Sprintf("%T", tx), - ) - continue - } - - txHashes = append(txHashes, txHash) + for _, ethLog := range ethLogs { + filtered := FilterLogs(ethLog, f.criteria.FromBlock, f.criteria.ToBlock, f.criteria.Addresses, f.criteria.Topics) + logs = append(logs, filtered...) } - - logsMatched := f.checkMatches(txHashes) - logs = append(logs, logsMatched...) } - return logs, nil } @@ -207,21 +186,50 @@ func (f *Filter) blockLogs(header *ethtypes.Header) ([]*ethtypes.Log, error) { return logs, nil } -// checkMatches checks if the logs from the a list of transactions transaction -// contain any log events that match the filter criteria. This function is -// called when the bloom filter signals a potential match. -func (f *Filter) checkMatches(transactions []common.Hash) []*ethtypes.Log { - unfiltered := []*ethtypes.Log{} - for _, tx := range transactions { - logs, err := f.backend.GetTransactionLogs(tx) - if err != nil { - // ignore error if transaction didn't set any logs (eg: when tx type is not - // MsgEthereumTx or MsgEthermint) +func createBloomFilters(filters [][][]byte) [][]BloomIV { + bloomFilters := make([][]BloomIV, 0) + for _, filter := range filters { + // Gather the bit indexes of the filter rule, special casing the nil filter + if len(filter) == 0 { continue } - - unfiltered = append(unfiltered, logs...) + bloomIVs := make([]BloomIV, len(filter)) + for i, clause := range filter { + if clause == nil { + bloomIVs = nil + break + } + bloomIVs[i] = calcBloomIVs(clause) + } + // Accumulate the filter rules if no nil rule was within + if bloomIVs != nil { + bloomFilters = append(bloomFilters, bloomIVs) + } } + return bloomFilters +} - return FilterLogs(unfiltered, f.criteria.FromBlock, f.criteria.ToBlock, f.criteria.Addresses, f.criteria.Topics) +// calcBloomIVs returns BloomIV for the given data, +// revised from https://github.com/ethereum/go-ethereum/blob/401354976bb44f0ad4455ca1e0b5c0dc31d9a5f5/core/types/bloom9.go#L139 +func calcBloomIVs(data []byte) BloomIV { + hashbuf := make([]byte, 6) + biv := BloomIV{} + + sha := crypto.NewKeccakState() + sha.Reset() + sha.Write(data) + if _, err := sha.Read(hashbuf); err != nil { + panic(err) + } + + // The actual bits to flip + biv.V[0] = byte(1 << (hashbuf[1] & 0x7)) + biv.V[1] = byte(1 << (hashbuf[3] & 0x7)) + biv.V[2] = byte(1 << (hashbuf[5] & 0x7)) + // The indices for the bytes to OR in + biv.I[0] = ethtypes.BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf)&0x7ff)>>3) - 1 + biv.I[1] = ethtypes.BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf[2:])&0x7ff)>>3) - 1 + biv.I[2] = ethtypes.BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf[4:])&0x7ff)>>3) - 1 + + return biv } diff --git a/tests/rpc/rpc_test.go b/tests/rpc/rpc_test.go index 6436896013..1e57b831a8 100644 --- a/tests/rpc/rpc_test.go +++ b/tests/rpc/rpc_test.go @@ -34,8 +34,7 @@ const ( ) var ( - MODE = os.Getenv("MODE") - + MODE = os.Getenv("MODE") zeroString = "0x0" from = []byte{} ) @@ -911,3 +910,38 @@ func TestEth_GetBlockByNumber(t *testing.T) { require.Equal(t, "0x", block["extraData"].(string)) require.Equal(t, []interface{}{}, block["uncles"].([]interface{})) } + +func TestEth_GetLogs(t *testing.T) { + time.Sleep(time.Second) + + rpcRes := call(t, "eth_blockNumber", []string{}) + + var res hexutil.Uint64 + err := res.UnmarshalJSON(rpcRes.Result) + require.NoError(t, err) + + param := make([]map[string]interface{}, 1) + param[0] = make(map[string]interface{}) + param[0]["topics"] = []string{helloTopic, worldTopic} + param[0]["fromBlock"] = res.String() + + deployTestContractWithFunction(t) + + // get filter changes + logRes := call(t, "eth_getLogs", param) + + var logs []*ethtypes.Log + err = json.Unmarshal(logRes.Result, &logs) + require.NoError(t, err) + + require.Equal(t, 1, len(logs)) + + // filter log with address + param[0] = make(map[string]interface{}) + param[0]["address"] = "0x" + fmt.Sprintf("%x", from) + param[0]["fromBlock"] = res.String() + err = json.Unmarshal(logRes.Result, &logs) + require.NoError(t, err) + + require.Equal(t, 1, len(logs)) +} From f40c5299ed0efaf9f807ea7826203ecfb81ca47a Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 17 Sep 2021 21:51:19 -0400 Subject: [PATCH 2/5] fix lint --- ethereum/rpc/namespaces/eth/filters/filters.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ethereum/rpc/namespaces/eth/filters/filters.go b/ethereum/rpc/namespaces/eth/filters/filters.go index 989b784171..98155231b5 100644 --- a/ethereum/rpc/namespaces/eth/filters/filters.go +++ b/ethereum/rpc/namespaces/eth/filters/filters.go @@ -217,7 +217,9 @@ func calcBloomIVs(data []byte) BloomIV { sha := crypto.NewKeccakState() sha.Reset() - sha.Write(data) + if _, err := sha.Write(data); err != nil { + panic(err) + } if _, err := sha.Read(hashbuf); err != nil { panic(err) } From 586b56edc7b4c781e9e1c4f011a099a409a279b9 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Thu, 23 Sep 2021 10:49:31 -0400 Subject: [PATCH 3/5] error handling in calcBloomIVs --- .../rpc/namespaces/eth/filters/filters.go | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/ethereum/rpc/namespaces/eth/filters/filters.go b/ethereum/rpc/namespaces/eth/filters/filters.go index 98155231b5..787faec0a9 100644 --- a/ethereum/rpc/namespaces/eth/filters/filters.go +++ b/ethereum/rpc/namespaces/eth/filters/filters.go @@ -194,12 +194,23 @@ func createBloomFilters(filters [][][]byte) [][]BloomIV { continue } bloomIVs := make([]BloomIV, len(filter)) + + // Transform the filter rules (the addresses and topics) to the bloom index and value arrays + // So it can be used to compare with the bloom of the block header. If the rule has any nil + // clauses. The rule will be ignored. for i, clause := range filter { if clause == nil { bloomIVs = nil break } - bloomIVs[i] = calcBloomIVs(clause) + + iv, err := calcBloomIVs(clause) + if err != nil { + bloomIVs = nil + break + } + + bloomIVs[i] = iv } // Accumulate the filter rules if no nil rule was within if bloomIVs != nil { @@ -211,17 +222,17 @@ func createBloomFilters(filters [][][]byte) [][]BloomIV { // calcBloomIVs returns BloomIV for the given data, // revised from https://github.com/ethereum/go-ethereum/blob/401354976bb44f0ad4455ca1e0b5c0dc31d9a5f5/core/types/bloom9.go#L139 -func calcBloomIVs(data []byte) BloomIV { +func calcBloomIVs(data []byte) (BloomIV, error) { hashbuf := make([]byte, 6) biv := BloomIV{} sha := crypto.NewKeccakState() sha.Reset() if _, err := sha.Write(data); err != nil { - panic(err) + return BloomIV{}, err } if _, err := sha.Read(hashbuf); err != nil { - panic(err) + return BloomIV{}, err } // The actual bits to flip @@ -233,5 +244,5 @@ func calcBloomIVs(data []byte) BloomIV { biv.I[1] = ethtypes.BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf[2:])&0x7ff)>>3) - 1 biv.I[2] = ethtypes.BloomByteLength - uint((binary.BigEndian.Uint16(hashbuf[4:])&0x7ff)>>3) - 1 - return biv + return biv, nil } From d10a59eecd023d0a6e2b606983fb63d123b54002 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Thu, 23 Sep 2021 10:58:09 -0400 Subject: [PATCH 4/5] print error log in createBloomFilters --- ethereum/rpc/namespaces/eth/filters/filters.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ethereum/rpc/namespaces/eth/filters/filters.go b/ethereum/rpc/namespaces/eth/filters/filters.go index 787faec0a9..aa82a7ad0a 100644 --- a/ethereum/rpc/namespaces/eth/filters/filters.go +++ b/ethereum/rpc/namespaces/eth/filters/filters.go @@ -70,7 +70,7 @@ func NewRangeFilter(logger log.Logger, backend Backend, begin, end int64, addres Topics: topics, } - return newFilter(logger, backend, criteria, createBloomFilters(filtersBz)) + return newFilter(logger, backend, criteria, createBloomFilters(filtersBz, logger)) } // newFilter returns a new Filter @@ -186,7 +186,7 @@ func (f *Filter) blockLogs(header *ethtypes.Header) ([]*ethtypes.Log, error) { return logs, nil } -func createBloomFilters(filters [][][]byte) [][]BloomIV { +func createBloomFilters(filters [][][]byte, logger log.Logger) [][]BloomIV { bloomFilters := make([][]BloomIV, 0) for _, filter := range filters { // Gather the bit indexes of the filter rule, special casing the nil filter @@ -207,6 +207,7 @@ func createBloomFilters(filters [][][]byte) [][]BloomIV { iv, err := calcBloomIVs(clause) if err != nil { bloomIVs = nil + logger.Error("calcBloomIVs error: %v", err) break } From 62fa38d998db4793e9d3114a2bfad4b67a6fac81 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Thu, 23 Sep 2021 15:38:07 -0400 Subject: [PATCH 5/5] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb90a9ec3b..b27cd4fc77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (evm) [tharsis#461](https://github.com/tharsis/ethermint/pull/461) Increase performance of `StateDB` transaction log storage (r/w). * (evm) [tharsis#566](https://github.com/tharsis/ethermint/pull/566) Introduce `stateErr` store in `StateDB` to avoid meaningless operations if any error happened before +* (rpc/evm) [tharsis#587](https://github.com/tharsis/ethermint/pull/587) Apply bloom filter when query ethlogs with range of blocks ## [v0.5.0] - 2021-08-20