From ae4a67ab42025e51cd064a52b7b88d3c7430738d Mon Sep 17 00:00:00 2001 From: Emmanuel Date: Thu, 15 Feb 2024 10:19:45 -0300 Subject: [PATCH] Enhance functionality for some new features (#112) * make zcache ttl configurable * make robust addresses consolidation process optional * fix tests * fix linter * condense code into generic function * cover a missing case for addresses robust consolidation process --- actors/cache/impl/zcache.go | 20 ++++++++++--------- actors/common.go | 26 +++++++++++++++++++++---- factory.go | 6 +++--- parser/types.go | 12 +++++++++++- parser/v1/parser.go | 38 ++++++++++++++++++++++++++++--------- parser/v2/parser.go | 29 +++++++++++++++++++++------- parser_test.go | 26 ++++++++++++++++++++++--- 7 files changed, 121 insertions(+), 36 deletions(-) diff --git a/actors/cache/impl/zcache.go b/actors/cache/impl/zcache.go index 69e34d20..72a02e5b 100644 --- a/actors/cache/impl/zcache.go +++ b/actors/cache/impl/zcache.go @@ -3,8 +3,10 @@ package impl import ( "context" "fmt" - "github.com/zondax/fil-parser/actors/constants" "strings" + "time" + + "github.com/zondax/fil-parser/actors/constants" "github.com/filecoin-project/go-address" filTypes "github.com/filecoin-project/lotus/chain/types" @@ -20,9 +22,8 @@ const ( ZCacheImpl = "zcache" ZCacheLocalOnly = "in-memory" ZCacheCombined = "combined" - NoTtl = -1 - NotExpiringTtl = -1 PrefixSplitter = "/" + NotExpiringTtl = -1 ) // ZCache In-Memory database @@ -32,7 +33,7 @@ type ZCache struct { shortRobustMap zcache.ZCache logger *zap.Logger cacheType string - ttl int + ttl time.Duration } func (m *ZCache) NewImpl(source common.DataSource, logger *zap.Logger) error { @@ -43,9 +44,10 @@ func (m *ZCache) NewImpl(source common.DataSource, logger *zap.Logger) error { // remote best effort, as the remote cache will fail. However, the cache will // work anyway cacheConfig := source.Config.Cache + if cacheConfig == nil { m.cacheType = ZCacheLocalOnly - m.ttl = NoTtl + m.ttl = NotExpiringTtl if m.robustShortMap, err = zcache.NewLocalCache(&zcache.LocalConfig{Prefix: Robust2ShortMapPrefix, Logger: m.logger}); err != nil { return fmt.Errorf("error creating robustShortMap for local zcache, err: %s", err) @@ -55,7 +57,7 @@ func (m *ZCache) NewImpl(source common.DataSource, logger *zap.Logger) error { } } else { m.cacheType = ZCacheCombined - m.ttl = cacheConfig.GlobalTtlSeconds + m.ttl = time.Second * time.Duration(cacheConfig.GlobalTtlSeconds) prefix := "" if cacheConfig.GlobalPrefix != "" { @@ -193,7 +195,7 @@ func (m *ZCache) storeRobustShort(robust string, short string) { // Possible ZCache types can be Local or Combined. Both types set the TTL at instantiation time // The ttl here is pointless ctx := context.Background() - _ = m.robustShortMap.Set(ctx, robust, short, NotExpiringTtl) + _ = m.robustShortMap.Set(ctx, robust, short, m.ttl) } func (m *ZCache) storeShortRobust(short string, robust string) { @@ -205,7 +207,7 @@ func (m *ZCache) storeShortRobust(short string, robust string) { // Possible ZCache types can be Local or Combined. Both types set the TTL at instantiation time // The ttl here is pointless ctx := context.Background() - _ = m.shortRobustMap.Set(ctx, short, robust, NotExpiringTtl) + _ = m.shortRobustMap.Set(ctx, short, robust, m.ttl) } func (m *ZCache) StoreAddressInfo(info types.AddressInfo) { @@ -232,7 +234,7 @@ func (m *ZCache) storeActorCode(shortAddress string, cid string) { // Possible ZCache types can be Local or Combined. Both types set the TTL at instantiation time // The ttl here is pointless ctx := context.Background() - _ = m.shortCidMap.Set(ctx, shortAddress, cid, NotExpiringTtl) + _ = m.shortCidMap.Set(ctx, shortAddress, cid, m.ttl) } func (m *ZCache) tryToGetF4Address(address address.Address) string { diff --git a/actors/common.go b/actors/common.go index 57b573f9..1d7db8f5 100644 --- a/actors/common.go +++ b/actors/common.go @@ -3,6 +3,8 @@ package actors import ( "bytes" "encoding/hex" + "fmt" + "github.com/filecoin-project/go-address" "github.com/zondax/fil-parser/actors/cache" "github.com/zondax/fil-parser/actors/cache/impl/common" @@ -44,15 +46,31 @@ func (p *ActorParser) emptyParamsAndReturn() (map[string]interface{}, error) { return make(map[string]interface{}), nil } -func EnsureRobustAddress(address address.Address, actorCache *cache.ActorsCache, logger *zap.Logger) string { +func ConsolidateRobustAddresses(from, to address.Address, actorCache *cache.ActorsCache, logger *zap.Logger, config *parser.ConsolidateAddressesToRobust) (string, string, error) { + var err error + txFrom := from.String() + txTo := to.String() + if config != nil && config.Enable { + if txFrom, err = EnsureRobustAddress(from, actorCache, logger); err != nil && !config.BestEffort { + return "", "", err + } + if txTo, err = EnsureRobustAddress(to, actorCache, logger); err != nil && !config.BestEffort { + return "", "", err + } + } + + return txFrom, txTo, nil +} + +func EnsureRobustAddress(address address.Address, actorCache *cache.ActorsCache, logger *zap.Logger) (string, error) { if isRobust, _ := common.IsRobustAddress(address); isRobust { - return address.String() + return address.String(), nil } robustAddress, err := actorCache.GetRobustAddress(address) if err != nil { logger.Sugar().Warnf("Error converting address to robust format: %v", err) - return address.String() // Fallback + return address.String(), fmt.Errorf("error converting address to robust format: %v", err) // Fallback } - return robustAddress + return robustAddress, nil } diff --git a/factory.go b/factory.go index b984ee4d..7584c496 100644 --- a/factory.go +++ b/factory.go @@ -41,7 +41,7 @@ type Parser interface { IsNodeVersionSupported(ver string) bool } -func NewFilecoinParser(lib *rosettaFilecoinLib.RosettaConstructionFilecoin, cacheSource common.DataSource, logger *zap.Logger) (*FilecoinParser, error) { +func NewFilecoinParser(lib *rosettaFilecoinLib.RosettaConstructionFilecoin, cacheSource common.DataSource, logger *zap.Logger, config *parser.FilecoinParserConfig) (*FilecoinParser, error) { logger = logger2.GetSafeLogger(logger) actorsCache, err := cache.SetupActorsCache(cacheSource, logger) if err != nil { @@ -50,8 +50,8 @@ func NewFilecoinParser(lib *rosettaFilecoinLib.RosettaConstructionFilecoin, cach } helper := helper2.NewHelper(lib, actorsCache, logger) - parserV1 := v1.NewParser(helper, logger) - parserV2 := v2.NewParser(helper, logger) + parserV1 := v1.NewParser(helper, logger, config) + parserV2 := v2.NewParser(helper, logger, config) return &FilecoinParser{ parserV1: parserV1, diff --git a/parser/types.go b/parser/types.go index f3cf5353..7e9272ba 100644 --- a/parser/types.go +++ b/parser/types.go @@ -2,15 +2,25 @@ package parser import ( "encoding/json" + "math/big" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/lotus/api" "github.com/ipfs/go-cid" - "math/big" ) +type FilecoinParserConfig struct { + ConsolidateAddressesToRobust ConsolidateAddressesToRobust +} + +type ConsolidateAddressesToRobust struct { + Enable bool + BestEffort bool +} + type ControlAddress struct { Owner string `json:"owner"` Worker string `json:"worker"` diff --git a/parser/v1/parser.go b/parser/v1/parser.go index fe665256..b7bdab5b 100644 --- a/parser/v1/parser.go +++ b/parser/v1/parser.go @@ -3,11 +3,12 @@ package v1 import ( "encoding/json" "errors" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/api" "math/big" "strings" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + "github.com/bytedance/sonic" filTypes "github.com/filecoin-project/lotus/chain/types" "github.com/google/uuid" @@ -31,14 +32,16 @@ type Parser struct { addresses *types.AddressInfoMap helper *helper.Helper logger *zap.Logger + config *parser.FilecoinParserConfig } -func NewParser(helper *helper.Helper, logger *zap.Logger) *Parser { +func NewParser(helper *helper.Helper, logger *zap.Logger, config *parser.FilecoinParserConfig) *Parser { return &Parser{ actorParser: actors.NewActorParser(helper, logger), addresses: types.NewAddressInfoMap(), helper: helper, logger: logger2.GetSafeLogger(logger), + config: config, } } @@ -95,6 +98,16 @@ func (p *Parser) ParseTransactions(traces []byte, tipset *types.ExtendedTipSet, } messageUuid := tools.BuildMessageId(tipsetCid, blockCid, trace.MsgCid.String(), trace.Msg.Cid().String(), uuid.Nil.String()) + config := &parser.ConsolidateAddressesToRobust{} + if p.config != nil { + config = &p.config.ConsolidateAddressesToRobust + } + + txFrom, txTo, err := actors.ConsolidateRobustAddresses(trace.Msg.From, trace.Msg.To, p.helper.GetActorsCache(), p.logger, config) + if err != nil { + return nil, nil, err + } + badTx := &types.Transaction{ TxBasicBlockData: types.TxBasicBlockData{ BasicBlockData: types.BasicBlockData{ @@ -106,8 +119,8 @@ func (p *Parser) ParseTransactions(traces []byte, tipset *types.ExtendedTipSet, Id: messageUuid, ParentId: uuid.Nil.String(), TxCid: trace.MsgCid.String(), - TxFrom: trace.Msg.From.String(), - TxTo: trace.Msg.To.String(), + TxFrom: txFrom, + TxTo: txTo, TxType: txType, Amount: trace.Msg.Value.Int, GasUsed: uint64(trace.MsgRct.GasUsed), @@ -242,8 +255,15 @@ func (p *Parser) parseTrace(trace typesV1.ExecutionTraceV1, mainMsgCid cid.Cid, messageUuid := tools.BuildMessageId(tipsetCid, blockCid, mainMsgCid.String(), trace.Msg.Cid().String(), parentId) - txFromRobust := actors.EnsureRobustAddress(trace.Msg.From, p.helper.GetActorsCache(), p.logger) - txToRobust := actors.EnsureRobustAddress(trace.Msg.To, p.helper.GetActorsCache(), p.logger) + config := &parser.ConsolidateAddressesToRobust{} + if p.config != nil { + config = &p.config.ConsolidateAddressesToRobust + } + + txFrom, txTo, err := actors.ConsolidateRobustAddresses(trace.Msg.From, trace.Msg.To, p.helper.GetActorsCache(), p.logger, config) + if err != nil { + return nil, err + } tx := &types.Transaction{ TxBasicBlockData: types.TxBasicBlockData{ @@ -257,8 +277,8 @@ func (p *Parser) parseTrace(trace typesV1.ExecutionTraceV1, mainMsgCid cid.Cid, Id: messageUuid, TxTimestamp: parser.GetTimestamp(tipset.MinTimestamp()), TxCid: mainMsgCid.String(), - TxFrom: txFromRobust, - TxTo: txToRobust, + TxFrom: txFrom, + TxTo: txTo, Amount: trace.Msg.Value.Int, Status: parser.GetExitCodeStatus(trace.MsgRct.ExitCode), TxType: txType, diff --git a/parser/v2/parser.go b/parser/v2/parser.go index 0ec2b4c0..d7025797 100644 --- a/parser/v2/parser.go +++ b/parser/v2/parser.go @@ -3,11 +3,16 @@ package v2 import ( "encoding/json" "errors" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/lotus/api" "math/big" "strings" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/api" + + "github.com/bytedance/sonic" + filTypes "github.com/filecoin-project/lotus/chain/types" + "github.com/google/uuid" + "github.com/ipfs/go-cid" "github.com/zondax/fil-parser/actors" logger2 "github.com/zondax/fil-parser/logger" "github.com/zondax/fil-parser/parser" @@ -32,14 +37,16 @@ type Parser struct { addresses *types.AddressInfoMap helper *helper.Helper logger *zap.Logger + config *parser.FilecoinParserConfig } -func NewParser(helper *helper.Helper, logger *zap.Logger) *Parser { +func NewParser(helper *helper.Helper, logger *zap.Logger, config *parser.FilecoinParserConfig) *Parser { return &Parser{ actorParser: actors.NewActorParser(helper, logger), addresses: types.NewAddressInfoMap(), helper: helper, logger: logger2.GetSafeLogger(logger), + config: config, } } @@ -217,8 +224,16 @@ func (p *Parser) parseTrace(trace typesV2.ExecutionTraceV2, mainMsgCid cid.Cid, tipsetCid := tipset.GetCidString() messageUuid := tools.BuildMessageId(tipsetCid, blockCid, mainMsgCid.String(), msgCid, parentId) - txFromRobust := actors.EnsureRobustAddress(trace.Msg.From, p.helper.GetActorsCache(), p.logger) - txToRobust := actors.EnsureRobustAddress(trace.Msg.To, p.helper.GetActorsCache(), p.logger) + + config := &parser.ConsolidateAddressesToRobust{} + if p.config != nil { + config = &p.config.ConsolidateAddressesToRobust + } + + txFrom, txTo, err := actors.ConsolidateRobustAddresses(trace.Msg.From, trace.Msg.To, p.helper.GetActorsCache(), p.logger, config) + if err != nil { + return nil, err + } tx := &types.Transaction{ TxBasicBlockData: types.TxBasicBlockData{ @@ -232,8 +247,8 @@ func (p *Parser) parseTrace(trace typesV2.ExecutionTraceV2, mainMsgCid cid.Cid, Id: messageUuid, TxTimestamp: parser.GetTimestamp(tipset.MinTimestamp()), TxCid: mainMsgCid.String(), - TxFrom: txFromRobust, - TxTo: txToRobust, + TxFrom: txFrom, + TxTo: txTo, Amount: trace.Msg.Value.Int, Status: parser.GetExitCodeStatus(trace.MsgRct.ExitCode), TxType: txType, diff --git a/parser_test.go b/parser_test.go index e6e7de64..9ffc377c 100644 --- a/parser_test.go +++ b/parser_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/zondax/fil-parser/parser" + "go.uber.org/zap" "github.com/filecoin-project/lotus/api" @@ -223,7 +225,13 @@ func TestParser_ParseTransactions(t *testing.T) { logger, err := zap.NewDevelopment() require.NoError(t, err) - p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger) + config := &parser.FilecoinParserConfig{ + ConsolidateAddressesToRobust: parser.ConsolidateAddressesToRobust{ + Enable: true, + BestEffort: true, + }, + } + p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger, config) require.NoError(t, err) txs, adds, err := p.ParseTransactions(traces, tipset, ethlogs, types.BlockMetadata{NodeInfo: types.NodeInfo{NodeMajorMinorVersion: tt.version}}) require.NoError(t, err) @@ -272,7 +280,13 @@ func TestParser_GetBaseFee(t *testing.T) { logger, err := zap.NewDevelopment() require.NoError(t, err) - p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger) + config := &parser.FilecoinParserConfig{ + ConsolidateAddressesToRobust: parser.ConsolidateAddressesToRobust{ + Enable: true, + BestEffort: true, + }, + } + p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger, config) require.NoError(t, err) baseFee, err := p.GetBaseFee(traces, types.BlockMetadata{}, tipset) require.NoError(t, err) @@ -315,7 +329,13 @@ func TestParser_InDepthCompare(t *testing.T) { logger, err := zap.NewDevelopment() require.NoError(t, err) - p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger) + config := &parser.FilecoinParserConfig{ + ConsolidateAddressesToRobust: parser.ConsolidateAddressesToRobust{ + Enable: true, + BestEffort: true, + }, + } + p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger, config) require.NoError(t, err) v1Txs, v1Adds, err := p.ParseTransactions(traces, tipset, ethlogs, types.BlockMetadata{NodeInfo: types.NodeInfo{NodeMajorMinorVersion: "v1.22"}}) require.NoError(t, err)