Skip to content

Commit

Permalink
Enhance functionality for some new features (#112)
Browse files Browse the repository at this point in the history 
* make zcache ttl configurable

* make robust addresses consolidation process optional

* fix tests

* fix linter

* condense code into generic function

* cover a missing case for addresses robust consolidation process
  • Loading branch information
@emmanuelm41
emmanuelm41 committed Apr 7, 2024
1 parent e34f728 commit ae4a67a
Show file tree
Hide file tree
Showing 7 changed files with 121 additions and 36 deletions.
20 changes: 11 additions & 9 deletions actors/cache/impl/zcache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@ package impl
import (
"context"
"fmt"
"github.com/zondax/fil-parser/actors/constants"
"strings"
"time"

"github.com/zondax/fil-parser/actors/constants"

"github.com/filecoin-project/go-address"
filTypes "github.com/filecoin-project/lotus/chain/types"
Expand All @@ -20,9 +22,8 @@ const (
ZCacheImpl = "zcache"
ZCacheLocalOnly = "in-memory"
ZCacheCombined = "combined"
NoTtl = -1
NotExpiringTtl = -1
PrefixSplitter = "/"
NotExpiringTtl = -1
)

// ZCache In-Memory database
Expand All @@ -32,7 +33,7 @@ type ZCache struct {
shortRobustMap zcache.ZCache
logger *zap.Logger
cacheType string
ttl int
ttl time.Duration
}

func (m *ZCache) NewImpl(source common.DataSource, logger *zap.Logger) error {
Expand All @@ -43,9 +44,10 @@ func (m *ZCache) NewImpl(source common.DataSource, logger *zap.Logger) error {
// remote best effort, as the remote cache will fail. However, the cache will
// work anyway
cacheConfig := source.Config.Cache

if cacheConfig == nil {
m.cacheType = ZCacheLocalOnly
m.ttl = NoTtl
m.ttl = NotExpiringTtl

if m.robustShortMap, err = zcache.NewLocalCache(&zcache.LocalConfig{Prefix: Robust2ShortMapPrefix, Logger: m.logger}); err != nil {
return fmt.Errorf("error creating robustShortMap for local zcache, err: %s", err)
Expand All @@ -55,7 +57,7 @@ func (m *ZCache) NewImpl(source common.DataSource, logger *zap.Logger) error {
}
} else {
m.cacheType = ZCacheCombined
m.ttl = cacheConfig.GlobalTtlSeconds
m.ttl = time.Second * time.Duration(cacheConfig.GlobalTtlSeconds)

prefix := ""
if cacheConfig.GlobalPrefix != "" {
Expand Down Expand Up @@ -193,7 +195,7 @@ func (m *ZCache) storeRobustShort(robust string, short string) {
// Possible ZCache types can be Local or Combined. Both types set the TTL at instantiation time
// The ttl here is pointless
ctx := context.Background()
_ = m.robustShortMap.Set(ctx, robust, short, NotExpiringTtl)
_ = m.robustShortMap.Set(ctx, robust, short, m.ttl)
}

func (m *ZCache) storeShortRobust(short string, robust string) {
Expand All @@ -205,7 +207,7 @@ func (m *ZCache) storeShortRobust(short string, robust string) {
// Possible ZCache types can be Local or Combined. Both types set the TTL at instantiation time
// The ttl here is pointless
ctx := context.Background()
_ = m.shortRobustMap.Set(ctx, short, robust, NotExpiringTtl)
_ = m.shortRobustMap.Set(ctx, short, robust, m.ttl)
}

func (m *ZCache) StoreAddressInfo(info types.AddressInfo) {
Expand All @@ -232,7 +234,7 @@ func (m *ZCache) storeActorCode(shortAddress string, cid string) {
// Possible ZCache types can be Local or Combined. Both types set the TTL at instantiation time
// The ttl here is pointless
ctx := context.Background()
_ = m.shortCidMap.Set(ctx, shortAddress, cid, NotExpiringTtl)
_ = m.shortCidMap.Set(ctx, shortAddress, cid, m.ttl)
}

func (m *ZCache) tryToGetF4Address(address address.Address) string {
Expand Down
26 changes: 22 additions & 4 deletions actors/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package actors
import (
"bytes"
"encoding/hex"
"fmt"

"github.com/filecoin-project/go-address"
"github.com/zondax/fil-parser/actors/cache"
"github.com/zondax/fil-parser/actors/cache/impl/common"
Expand Down Expand Up @@ -44,15 +46,31 @@ func (p *ActorParser) emptyParamsAndReturn() (map[string]interface{}, error) {
return make(map[string]interface{}), nil
}

func EnsureRobustAddress(address address.Address, actorCache *cache.ActorsCache, logger *zap.Logger) string {
func ConsolidateRobustAddresses(from, to address.Address, actorCache *cache.ActorsCache, logger *zap.Logger, config *parser.ConsolidateAddressesToRobust) (string, string, error) {
var err error
txFrom := from.String()
txTo := to.String()
if config != nil && config.Enable {
if txFrom, err = EnsureRobustAddress(from, actorCache, logger); err != nil && !config.BestEffort {
return "", "", err
}
if txTo, err = EnsureRobustAddress(to, actorCache, logger); err != nil && !config.BestEffort {
return "", "", err
}
}

return txFrom, txTo, nil
}

func EnsureRobustAddress(address address.Address, actorCache *cache.ActorsCache, logger *zap.Logger) (string, error) {
if isRobust, _ := common.IsRobustAddress(address); isRobust {
return address.String()
return address.String(), nil
}

robustAddress, err := actorCache.GetRobustAddress(address)
if err != nil {
logger.Sugar().Warnf("Error converting address to robust format: %v", err)
return address.String() // Fallback
return address.String(), fmt.Errorf("error converting address to robust format: %v", err) // Fallback
}
return robustAddress
return robustAddress, nil
}
6 changes: 3 additions & 3 deletions factory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ type Parser interface {
IsNodeVersionSupported(ver string) bool
}

func NewFilecoinParser(lib *rosettaFilecoinLib.RosettaConstructionFilecoin, cacheSource common.DataSource, logger *zap.Logger) (*FilecoinParser, error) {
func NewFilecoinParser(lib *rosettaFilecoinLib.RosettaConstructionFilecoin, cacheSource common.DataSource, logger *zap.Logger, config *parser.FilecoinParserConfig) (*FilecoinParser, error) {
logger = logger2.GetSafeLogger(logger)
actorsCache, err := cache.SetupActorsCache(cacheSource, logger)
if err != nil {
Expand All @@ -50,8 +50,8 @@ func NewFilecoinParser(lib *rosettaFilecoinLib.RosettaConstructionFilecoin, cach
}

helper := helper2.NewHelper(lib, actorsCache, logger)
parserV1 := v1.NewParser(helper, logger)
parserV2 := v2.NewParser(helper, logger)
parserV1 := v1.NewParser(helper, logger, config)
parserV2 := v2.NewParser(helper, logger, config)

return &FilecoinParser{
parserV1: parserV1,
Expand Down
12 changes: 11 additions & 1 deletion parser/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,25 @@ package parser

import (
"encoding/json"
"math/big"

"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/cbor"
"github.com/filecoin-project/go-state-types/exitcode"
"github.com/filecoin-project/lotus/api"
"github.com/ipfs/go-cid"
"math/big"
)

type FilecoinParserConfig struct {
ConsolidateAddressesToRobust ConsolidateAddressesToRobust
}

type ConsolidateAddressesToRobust struct {
Enable bool
BestEffort bool
}

type ControlAddress struct {
Owner string `json:"owner"`
Worker string `json:"worker"`
Expand Down
38 changes: 29 additions & 9 deletions parser/v1/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ package v1
import (
"encoding/json"
"errors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"math/big"
"strings"

"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"

"github.com/bytedance/sonic"
filTypes "github.com/filecoin-project/lotus/chain/types"
"github.com/google/uuid"
Expand All @@ -31,14 +32,16 @@ type Parser struct {
addresses *types.AddressInfoMap
helper *helper.Helper
logger *zap.Logger
config *parser.FilecoinParserConfig
}

func NewParser(helper *helper.Helper, logger *zap.Logger) *Parser {
func NewParser(helper *helper.Helper, logger *zap.Logger, config *parser.FilecoinParserConfig) *Parser {
return &Parser{
actorParser: actors.NewActorParser(helper, logger),
addresses: types.NewAddressInfoMap(),
helper: helper,
logger: logger2.GetSafeLogger(logger),
config: config,
}
}

Expand Down Expand Up @@ -95,6 +98,16 @@ func (p *Parser) ParseTransactions(traces []byte, tipset *types.ExtendedTipSet,
}
messageUuid := tools.BuildMessageId(tipsetCid, blockCid, trace.MsgCid.String(), trace.Msg.Cid().String(), uuid.Nil.String())

config := &parser.ConsolidateAddressesToRobust{}
if p.config != nil {
config = &p.config.ConsolidateAddressesToRobust
}

txFrom, txTo, err := actors.ConsolidateRobustAddresses(trace.Msg.From, trace.Msg.To, p.helper.GetActorsCache(), p.logger, config)
if err != nil {
return nil, nil, err
}

badTx := &types.Transaction{
TxBasicBlockData: types.TxBasicBlockData{
BasicBlockData: types.BasicBlockData{
Expand All @@ -106,8 +119,8 @@ func (p *Parser) ParseTransactions(traces []byte, tipset *types.ExtendedTipSet,
Id: messageUuid,
ParentId: uuid.Nil.String(),
TxCid: trace.MsgCid.String(),
TxFrom: trace.Msg.From.String(),
TxTo: trace.Msg.To.String(),
TxFrom: txFrom,
TxTo: txTo,
TxType: txType,
Amount: trace.Msg.Value.Int,
GasUsed: uint64(trace.MsgRct.GasUsed),
Expand Down Expand Up @@ -242,8 +255,15 @@ func (p *Parser) parseTrace(trace typesV1.ExecutionTraceV1, mainMsgCid cid.Cid,

messageUuid := tools.BuildMessageId(tipsetCid, blockCid, mainMsgCid.String(), trace.Msg.Cid().String(), parentId)

txFromRobust := actors.EnsureRobustAddress(trace.Msg.From, p.helper.GetActorsCache(), p.logger)
txToRobust := actors.EnsureRobustAddress(trace.Msg.To, p.helper.GetActorsCache(), p.logger)
config := &parser.ConsolidateAddressesToRobust{}
if p.config != nil {
config = &p.config.ConsolidateAddressesToRobust
}

txFrom, txTo, err := actors.ConsolidateRobustAddresses(trace.Msg.From, trace.Msg.To, p.helper.GetActorsCache(), p.logger, config)
if err != nil {
return nil, err
}

tx := &types.Transaction{
TxBasicBlockData: types.TxBasicBlockData{
Expand All @@ -257,8 +277,8 @@ func (p *Parser) parseTrace(trace typesV1.ExecutionTraceV1, mainMsgCid cid.Cid,
Id: messageUuid,
TxTimestamp: parser.GetTimestamp(tipset.MinTimestamp()),
TxCid: mainMsgCid.String(),
TxFrom: txFromRobust,
TxTo: txToRobust,
TxFrom: txFrom,
TxTo: txTo,
Amount: trace.Msg.Value.Int,
Status: parser.GetExitCodeStatus(trace.MsgRct.ExitCode),
TxType: txType,
Expand Down
29 changes: 22 additions & 7 deletions parser/v2/parser.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,16 @@ package v2
import (
"encoding/json"
"errors"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"
"math/big"
"strings"

"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/api"

"github.com/bytedance/sonic"
filTypes "github.com/filecoin-project/lotus/chain/types"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
"github.com/zondax/fil-parser/actors"
logger2 "github.com/zondax/fil-parser/logger"
"github.com/zondax/fil-parser/parser"
Expand All @@ -32,14 +37,16 @@ type Parser struct {
addresses *types.AddressInfoMap
helper *helper.Helper
logger *zap.Logger
config *parser.FilecoinParserConfig
}

func NewParser(helper *helper.Helper, logger *zap.Logger) *Parser {
func NewParser(helper *helper.Helper, logger *zap.Logger, config *parser.FilecoinParserConfig) *Parser {
return &Parser{
actorParser: actors.NewActorParser(helper, logger),
addresses: types.NewAddressInfoMap(),
helper: helper,
logger: logger2.GetSafeLogger(logger),
config: config,
}
}

Expand Down Expand Up @@ -217,8 +224,16 @@ func (p *Parser) parseTrace(trace typesV2.ExecutionTraceV2, mainMsgCid cid.Cid,

tipsetCid := tipset.GetCidString()
messageUuid := tools.BuildMessageId(tipsetCid, blockCid, mainMsgCid.String(), msgCid, parentId)
txFromRobust := actors.EnsureRobustAddress(trace.Msg.From, p.helper.GetActorsCache(), p.logger)
txToRobust := actors.EnsureRobustAddress(trace.Msg.To, p.helper.GetActorsCache(), p.logger)

config := &parser.ConsolidateAddressesToRobust{}
if p.config != nil {
config = &p.config.ConsolidateAddressesToRobust
}

txFrom, txTo, err := actors.ConsolidateRobustAddresses(trace.Msg.From, trace.Msg.To, p.helper.GetActorsCache(), p.logger, config)
if err != nil {
return nil, err
}

tx := &types.Transaction{
TxBasicBlockData: types.TxBasicBlockData{
Expand All @@ -232,8 +247,8 @@ func (p *Parser) parseTrace(trace typesV2.ExecutionTraceV2, mainMsgCid cid.Cid,
Id: messageUuid,
TxTimestamp: parser.GetTimestamp(tipset.MinTimestamp()),
TxCid: mainMsgCid.String(),
TxFrom: txFromRobust,
TxTo: txToRobust,
TxFrom: txFrom,
TxTo: txTo,
Amount: trace.Msg.Value.Int,
Status: parser.GetExitCodeStatus(trace.MsgRct.ExitCode),
TxType: txType,
Expand Down
26 changes: 23 additions & 3 deletions parser_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (

"github.com/stretchr/testify/assert"

"github.com/zondax/fil-parser/parser"

"go.uber.org/zap"

"github.com/filecoin-project/lotus/api"
Expand Down Expand Up @@ -223,7 +225,13 @@ func TestParser_ParseTransactions(t *testing.T) {
logger, err := zap.NewDevelopment()
require.NoError(t, err)

p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger)
config := &parser.FilecoinParserConfig{
ConsolidateAddressesToRobust: parser.ConsolidateAddressesToRobust{
Enable: true,
BestEffort: true,
},
}
p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger, config)
require.NoError(t, err)
txs, adds, err := p.ParseTransactions(traces, tipset, ethlogs, types.BlockMetadata{NodeInfo: types.NodeInfo{NodeMajorMinorVersion: tt.version}})
require.NoError(t, err)
Expand Down Expand Up @@ -272,7 +280,13 @@ func TestParser_GetBaseFee(t *testing.T) {
logger, err := zap.NewDevelopment()
require.NoError(t, err)

p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger)
config := &parser.FilecoinParserConfig{
ConsolidateAddressesToRobust: parser.ConsolidateAddressesToRobust{
Enable: true,
BestEffort: true,
},
}
p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger, config)
require.NoError(t, err)
baseFee, err := p.GetBaseFee(traces, types.BlockMetadata{}, tipset)
require.NoError(t, err)
Expand Down Expand Up @@ -315,7 +329,13 @@ func TestParser_InDepthCompare(t *testing.T) {
logger, err := zap.NewDevelopment()
require.NoError(t, err)

p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger)
config := &parser.FilecoinParserConfig{
ConsolidateAddressesToRobust: parser.ConsolidateAddressesToRobust{
Enable: true,
BestEffort: true,
},
}
p, err := NewFilecoinParser(lib, getCacheDataSource(t, tt.url), logger, config)
require.NoError(t, err)
v1Txs, v1Adds, err := p.ParseTransactions(traces, tipset, ethlogs, types.BlockMetadata{NodeInfo: types.NodeInfo{NodeMajorMinorVersion: "v1.22"}})
require.NoError(t, err)
Expand Down

0 comments on commit ae4a67a

Please sign in to comment.