diff --git a/go.mod b/go.mod index 7457cc3d..ca46e23b 100644 --- a/go.mod +++ b/go.mod @@ -149,7 +149,7 @@ require ( github.com/multiformats/go-multiaddr v0.12.3 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect - github.com/multiformats/go-multicodec v0.9.0 + github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect diff --git a/parser_test.go b/parser_test.go index b8f38187..6b50b4ce 100644 --- a/parser_test.go +++ b/parser_test.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "math" "math/big" "math/rand" "net/http" @@ -18,15 +19,18 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/stretchr/testify/assert" "go.uber.org/zap" "github.com/filecoin-project/go-address" + filBig "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/lotus/api" filTypes "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" + cidLink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/zondax/fil-parser/actors/cache/impl/common" v1 "github.com/zondax/fil-parser/parser/v1" v2 "github.com/zondax/fil-parser/parser/v2" @@ -628,6 +632,30 @@ func TestParser_ParseEvents_FVM_FromTraceFile(t *testing.T) { } } +func buildCidLink(cid cid.Cid) datamodel.Link { + return cidLink.Link{Cid: cid} +} + +func ipldEncode(t *testing.T, builder datamodel.NodeBuilder, data any) []byte { + var err error + + switch x := data.(type) { + case string: + err = builder.AssignString(x) + case []byte: + err = builder.AssignBytes(x) + case datamodel.Link: + err = builder.AssignLink(x) + case int64: + err = builder.AssignInt(x) + } + + require.NoError(t, err) + encoded, err := ipld.Encode(builder.Build(), dagcbor.Encode) + require.NoError(t, err) + return encoded +} + func TestParser_ParseNativeEvents_FVM(t *testing.T) { // we need any random number for the test //nolint:gosec @@ -645,17 +673,43 @@ func TestParser_ParseNativeEvents_FVM(t *testing.T) { parser, err := NewFilecoinParser(nil, getCacheDataSource(t, calibNextNodeUrl), logger) require.NoError(t, err) - ipldNodeBuilder := basicnode.Prototype.String.NewBuilder() - err = ipldNodeBuilder.AssignString("market_deals_event") - assert.NoError(t, err) - eventType, err := ipld.Encode(ipldNodeBuilder.Build(), dagcbor.Encode) - assert.NoError(t, err) + eventType := ipldEncode(t, basicnode.Prototype.String.NewBuilder(), "market_deals_event") + eventData := ipldEncode(t, basicnode.Prototype.Bytes.NewBuilder(), []byte("test_data")) - ipldNodeBuilder = basicnode.Prototype.Bytes.NewBuilder() - err = ipldNodeBuilder.AssignBytes([]byte("test data")) - assert.NoError(t, err) - eventData, err := ipld.Encode(ipldNodeBuilder.Build(), dagcbor.Encode) - assert.NoError(t, err) + // cid event data + eventCid, err := cid.Decode("baga6ea4seaqeyz6zikyr2bqbhy6mrocoqwagx45vlbpsbem7euqv5mf3hrvn2fy") + require.NoError(t, err) + link := buildCidLink(eventCid) + cidEventType := ipldEncode(t, basicnode.Prototype.String.NewBuilder(), "sector_activated") + cidEventData := ipldEncode(t, basicnode.Prototype.Link.NewBuilder(), link) + + // nullable cid event data + nullableCidEventType := ipldEncode(t, basicnode.Prototype.String.NewBuilder(), "sector_activated") + b := basicnode.Prototype__Any{}.NewBuilder() + err = b.AssignNull() + require.NoError(t, err) + nullableCidEventData, err := ipld.Encode(b.Build(), dagcbor.Encode) + require.NoError(t, err) + + // bigInt event data + bigInt, err := filBig.FromString("12345678901234567891234567890123456789012345678901234567890") + require.NoError(t, err) + bigIntEventType := ipldEncode(t, basicnode.Prototype.String.NewBuilder(), "verifier_balance") + tmp, err := bigInt.Bytes() + require.NoError(t, err) + bigIntEventData := ipldEncode(t, basicnode.Prototype.Bytes.NewBuilder(), tmp) + + largeInt := math.MaxInt64 + largeIntEventData := ipldEncode(t, basicnode.Prototype.Int.NewBuilder(), int64(largeInt)) + + smallInt := 10 + smallIntEventData := ipldEncode(t, basicnode.Prototype.Int.NewBuilder(), int64(smallInt)) + + negativeInt := -10 + negativeIntEventData := ipldEncode(t, basicnode.Prototype.Int.NewBuilder(), int64(negativeInt)) + + veryNegativeInt := math.MinInt64 + veryNegativeIntEventData := ipldEncode(t, basicnode.Prototype.Int.NewBuilder(), int64(veryNegativeInt)) tb := []struct { name string @@ -733,7 +787,262 @@ func TestParser_ParseNativeEvents_FVM(t *testing.T) { 1: { "flags": 3, "key": "data", - "value": "dGVzdCBkYXRh", + "value": "dGVzdF9kYXRh", + }, + }, + }, + { + name: "success native negative int event entries", + emitter: filAddress, + entries: []filTypes.EventEntry{ + { + Flags: 0x03, + Key: "$type", + Codec: 0x51, + Value: eventType, + }, + { + Flags: 0x03, + Key: "expiry", + Codec: 0x51, + Value: negativeIntEventData, + }, + }, + wantMetadata: map[int]map[string]any{ + 0: { + "flags": 3, + "key": "$type", + "value": "market_deals_event", + }, + 1: { + "flags": 3, + "key": "expiry", + "value": negativeInt, + }, + }, + }, + { + name: "success native very negative int event entries", + emitter: filAddress, + entries: []filTypes.EventEntry{ + { + Flags: 0x03, + Key: "$type", + Codec: 0x51, + Value: eventType, + }, + { + Flags: 0x03, + Key: "expiry", + Codec: 0x51, + Value: veryNegativeIntEventData, + }, + }, + wantMetadata: map[int]map[string]any{ + 0: { + "flags": 3, + "key": "$type", + "value": "market_deals_event", + }, + 1: { + "flags": 3, + "key": "expiry", + "value": fmt.Sprint(veryNegativeInt), + }, + }, + }, + { + name: "success native small int event entries", + emitter: filAddress, + entries: []filTypes.EventEntry{ + { + Flags: 0x03, + Key: "$type", + Codec: 0x51, + Value: eventType, + }, + { + Flags: 0x03, + Key: "expiry", + Codec: 0x51, + Value: smallIntEventData, + }, + }, + wantMetadata: map[int]map[string]any{ + 0: { + "flags": 3, + "key": "$type", + "value": "market_deals_event", + }, + 1: { + "flags": 3, + "key": "expiry", + "value": smallInt, + }, + }, + }, + { + name: "success native large int event entries", + emitter: filAddress, + entries: []filTypes.EventEntry{ + { + Flags: 0x03, + Key: "$type", + Codec: 0x51, + Value: eventType, + }, + { + Flags: 0x03, + Key: "expiry", + Codec: 0x51, + Value: largeIntEventData, + }, + }, + wantMetadata: map[int]map[string]any{ + 0: { + "flags": 3, + "key": "$type", + "value": "market_deals_event", + }, + 1: { + "flags": 3, + "key": "expiry", + "value": fmt.Sprint(largeInt), + }, + }, + }, + { + name: "success native bigInt event entries", + emitter: filAddress, + entries: []filTypes.EventEntry{ + { + Flags: 0x03, + Key: "$type", + Codec: 0x51, + Value: bigIntEventType, + }, + { + Flags: 0x03, + Key: "balance", + Codec: 0x51, + Value: bigIntEventData, + }, + }, + wantMetadata: map[int]map[string]any{ + 0: { + "flags": 3, + "key": "$type", + "value": "verifier_balance", + }, + 1: { + "flags": 3, + "key": "balance", + "value": "12345678901234567891234567890123456789012345678901234567890", + }, + }, + }, + { + name: "succes native cid event entries", + emitter: filAddress, + entries: []filTypes.EventEntry{ + { + Flags: 0x03, + Key: "$type", + Codec: 0x51, + Value: cidEventType, + }, + { + Flags: 0x03, + Key: "piece_cid", + Codec: 0x51, + Value: cidEventData, + }, + }, + wantMetadata: map[int]map[string]any{ + 0: { + "flags": 3, + "key": "$type", + "value": "sector_activated", + }, + 1: { + "flags": 3, + "key": "piece_cid", + "value": map[string]any{ + "/": "baga6ea4seaqeyz6zikyr2bqbhy6mrocoqwagx45vlbpsbem7euqv5mf3hrvn2fy", + }, + }, + }, + }, + { + name: "succes native nullable cid event entries", + emitter: filAddress, + entries: []filTypes.EventEntry{ + { + Flags: 0x03, + Key: "$type", + Codec: 0x51, + Value: nullableCidEventType, + }, + { + Flags: 0x03, + Key: "unsealed_cid", + Codec: 0x51, + Value: nullableCidEventData, + }, + }, + wantMetadata: map[int]map[string]any{ + 0: { + "flags": 3, + "key": "$type", + "value": "sector_activated", + }, + 1: { + "flags": 3, + "key": "unsealed_cid", + "value": nil, + }, + }, + }, + { + name: "succes native nullable cid and valid cid event entries", + emitter: filAddress, + entries: []filTypes.EventEntry{ + { + Flags: 0x03, + Key: "$type", + Codec: 0x51, + Value: nullableCidEventType, + }, + { + Flags: 0x03, + Key: "unsealed_cid", + Codec: 0x51, + Value: nullableCidEventData, + }, + { + Flags: 0x03, + Key: "piece_cid", + Codec: 0x51, + Value: cidEventData, + }, + }, + wantMetadata: map[int]map[string]any{ + 0: { + "flags": 3, + "key": "$type", + "value": "sector_activated", + }, + 1: { + "flags": 3, + "key": "unsealed_cid", + "value": nil, + }, + 2: { + "flags": 3, + "key": "piece_cid", + "value": map[string]any{ + "/": "baga6ea4seaqeyz6zikyr2bqbhy6mrocoqwagx45vlbpsbem7euqv5mf3hrvn2fy", + }, }, }, }, @@ -759,9 +1068,9 @@ func TestParser_ParseNativeEvents_FVM(t *testing.T) { fmt.Println(err) return } - assert.NoError(t, err) - assert.NotNil(t, events) - assert.NotEmpty(t, events.ParsedEvents) + require.NoError(t, err) + require.NotNil(t, events) + require.NotEmpty(t, events.ParsedEvents) gotMetadata := map[int]map[string]any{} err = json.Unmarshal([]byte(events.ParsedEvents[0].Metadata), &gotMetadata) @@ -774,8 +1083,8 @@ func TestParser_ParseNativeEvents_FVM(t *testing.T) { } assert.EqualValues(t, tipset.GetCidString(), events.ParsedEvents[0].TipsetCid) assert.EqualValues(t, tt.emitter.String(), events.ParsedEvents[0].Emitter) - if len(tt.entries) > 0 { // only check for the selector_id if we have entries in the test case - assert.EqualValues(t, "market_deals_event", events.ParsedEvents[0].SelectorID) + if len(tt.entries) > 0 { // only check for the selector_id if we have entries in the test case\ + assert.Regexp(t, "market_deals_event|sector_activated|verifier_balance", events.ParsedEvents[0].SelectorID) } // check if IDs are unique for all events diff --git a/tools/events/decode.go b/tools/events/decode.go new file mode 100644 index 00000000..22fa0809 --- /dev/null +++ b/tools/events/decode.go @@ -0,0 +1,93 @@ +package event_tools + +import ( + "fmt" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/datamodel" + basicnode "github.com/ipld/go-ipld-prime/node/basic" + "math" + "regexp" +) + +const ( + maxJSONNumber = math.MaxInt32 + minJSONNumber = math.MinInt32 +) + +var ( + // - https://github.com/filecoin-project/lotus/blob/6e13eac5d51f08d964f1338d9fab7cca42014e5c/documentation/en/actor-events-api.md?plain=1#L365 + cidRegex = regexp.MustCompile("cid") + // https://github.com/filecoin-project/lotus/blob/6e13eac5d51f08d964f1338d9fab7cca42014e5c/documentation/en/actor-events-api.md?plain=1#L112 + bigintRegex = regexp.MustCompile("balance") +) + +// decode does an ipld decode of the entry.Value using dagcbor +func decode(entry types.EventEntry) (datamodel.Node, error) { + n, err := ipld.Decode(entry.Value, dagcbor.Decode) + if err != nil { + return nil, fmt.Errorf("error ipld decode entry: %w ", err) + } + + if n.Kind() == datamodel.Kind_Int { + val, err := n.AsInt() + if err != nil { + return nil, fmt.Errorf("error ipld node to int : %w ", err) + } + if val > maxJSONNumber || val < minJSONNumber { + return basicnode.NewString(fmt.Sprint(val)), nil + } + } + + return n, nil +} + +// parseBigInt uses the filecoin-project big package to decode a node into a big.Int +// required for the verifier_balance event +// https://github.com/filecoin-project/lotus/blob/6e13eac5d51f08d964f1338d9fab7cca42014e5c/documentation/en/actor-events-api.md?plain=1#L112 +func parseBigInt(n datamodel.Node) (any, error) { + hexEncodedInt, err := n.AsBytes() + if err != nil { + return nil, fmt.Errorf("error converting ipld node to string: %w", err) + } + + bigInt, err := big.FromBytes(hexEncodedInt) + if err != nil { + return nil, fmt.Errorf("error converting hex encoded bigint to big.Int: %w", err) + } + return bigInt.String(), nil +} + +// parseCid parses an ipld node into the correct cid implementation. +// special cases include entries that have a CID as a value. +// CIDs are represented as an ipld.Link which needs an extra step of decoding the CID +// to get the correct JSON representation. +// Current edge case entry keys: unsealed-cid,piece-cid +// References: +// - https://github.com/filecoin-project/lotus/blob/6e13eac5d51f08d964f1338d9fab7cca42014e5c/documentation/en/actor-events-api.md?plain=1#L365 +func parseCid(n datamodel.Node) (any, error) { + if n.Kind() == datamodel.Kind_Null { + // nullable CIDs that show up in unsealed_cid are represented as Null + // - https://github.com/filecoin-project/lotus/blob/5dffc05a30894283287345d61b6578be7897ee4b/itests/direct_data_onboard_verified_test.go#L194 + return nil, nil + } + // - https://github.com/filecoin-project/lotus/blob/5dffc05a30894283287345d61b6578be7897ee4b/itests/direct_data_onboard_verified_test.go#L201 + if n.Kind() != datamodel.Kind_Link { + return nil, fmt.Errorf("unexpected datamodel kind for cid: %s ,expected: link", n.Kind()) + } + + link, err := n.AsLink() + if err != nil { + return nil, fmt.Errorf("error converting cid ipld node to link : %s : %w", n.Kind(), err) + } + + c, err := cid.Decode(link.String()) + if err != nil { + return nil, fmt.Errorf("error decoding %s to cid: %w", link.String(), err) + } + + return c, nil +} diff --git a/tools/events/events.go b/tools/events/events.go index 3071cab6..4daa5c58 100644 --- a/tools/events/events.go +++ b/tools/events/events.go @@ -13,10 +13,7 @@ import ( filTypes "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types/ethtypes" "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/datamodel" - "github.com/multiformats/go-multicodec" "github.com/zondax/fil-parser/parser/helper" "github.com/zondax/fil-parser/tools" "github.com/zondax/fil-parser/types" @@ -200,6 +197,10 @@ func buildEVMEventMetaData[T interface{ string | ethtypes.EthHash }](data []byte func parseNativeEventEntry(eventType string, entries []filTypes.EventEntry) (map[int]map[string]any, error) { parsedEntries := map[int]map[string]any{} + var ( + edgeCaseEntries []int + ) + for idx, entry := range entries { parsedEntry := map[string]any{ parsedEntryKey: entry.Key, @@ -223,14 +224,21 @@ func parseNativeEventEntry(eventType string, entries []filTypes.EventEntry) (map } parsedEntry["value"] = selectorHash.String() case types.EventTypeNative: - if entry.Codec != uint64(multicodec.Cbor) { - break - } - n, err := ipld.Decode(entry.Value, dagcbor.Decode) + var ( + err error + parsedValue datamodel.Node + ) + parsedValue, err = decode(entry) if err != nil { - return nil, fmt.Errorf("error ipld decode native event: %w ", err) + zap.S().Error("error ipld decode native event: ", err, entry.Key, entry.Codec, entry.Value) + return nil, fmt.Errorf("error decoding native event: %w ", err) + } + + // if the entry key is a CID or a bigInt, we need to decode the parsedValue further + if cidRegex.MatchString(entry.Key) || bigintRegex.MatchString(entry.Key) { + edgeCaseEntries = append(edgeCaseEntries, idx) } - parsedEntry[parsedEntryValue] = n + parsedEntry[parsedEntryValue] = parsedValue } } parsedEntry[parsedEntryFlags] = entry.Flags @@ -240,5 +248,30 @@ func parseNativeEventEntry(eventType string, entries []filTypes.EventEntry) (map parsedEntries[idx] = parsedEntry } + + // decode the entry values for the CIDs and BigInts + for _, idx := range edgeCaseEntries { + var ( + err error + data any + ) + + key := parsedEntries[idx][parsedEntryKey].(string) + value := parsedEntries[idx][parsedEntryValue].(datamodel.Node) + switch { + case cidRegex.MatchString(key): + data, err = parseCid(value) + case bigintRegex.MatchString(key): + data, err = parseBigInt(value) + default: + err = fmt.Errorf("unable to retrieve %s from evm event entry", key) + } + + if err != nil { + return nil, fmt.Errorf("error parsing native event: %w", err) + } + parsedEntries[idx][parsedEntryValue] = data + } + return parsedEntries, nil }