From f0ab1e1fc86d3790eea743dee6aa717886321adc Mon Sep 17 00:00:00 2001 From: jeff <113397187+cyberhorsey@users.noreply.github.com> Date: Tue, 17 Sep 2024 23:18:04 -0700 Subject: [PATCH] fix(eventindexer): remove nft metadata indexing, far too slowy and bug ridden right now (#18126) Co-authored-by: David --- .github/workflows/eventindexer.yml | 3 + packages/eventindexer/api/api.go | 6 - packages/eventindexer/errors.go | 4 - .../indexer/fetch_nft_metadata.go | 180 ------------------ .../indexer/index_erc20_transfers.go | 2 +- .../indexer/index_nft_transfers.go | 124 +----------- packages/eventindexer/indexer/indexer.go | 7 - ...270906208845_create_nft_metadata_table.sql | 23 --- ...6208846_alter_nft_metadata_add_indexes.sql | 10 - ...ft_metadata_table_update_symbol_length.sql | 9 - ...alter_nft_balances_add_metadata_column.sql | 14 -- ...08850_alter_nft_metadata_addtl_indexes.sql | 12 -- ...08851_alter_nft_balances_addtl_indexes.sql | 10 - packages/eventindexer/nft_balance.go | 17 +- packages/eventindexer/nft_metadata.go | 102 ---------- .../pkg/http/get_by_address_and_event_test.go | 2 +- .../get_count_by_address_and_event_test.go | 2 +- ...et_nft_balances_by_address_and_chain_id.go | 26 --- .../pkg/http/get_unique_proposers_test.go | 2 +- .../pkg/http/get_unique_provers_test.go | 2 +- packages/eventindexer/pkg/http/server.go | 7 - packages/eventindexer/pkg/http/server_test.go | 43 +---- .../pkg/mock/nft_metadata_repository.go | 57 ------ packages/eventindexer/pkg/repo/nft_balance.go | 6 +- .../eventindexer/pkg/repo/nft_balance_test.go | 18 -- .../eventindexer/pkg/repo/nft_metadata.go | 93 --------- packages/relayer/pkg/http/server_test.go | 10 - 27 files changed, 24 insertions(+), 767 deletions(-) delete mode 100644 packages/eventindexer/indexer/fetch_nft_metadata.go delete mode 100644 packages/eventindexer/migrations/20270906208845_create_nft_metadata_table.sql delete mode 100644 packages/eventindexer/migrations/20270906208846_alter_nft_metadata_add_indexes.sql delete mode 100644 packages/eventindexer/migrations/20270906208847_alter_nft_metadata_table_update_symbol_length.sql delete mode 100644 packages/eventindexer/migrations/20270906208848_alter_nft_balances_add_metadata_column.sql delete mode 100644 packages/eventindexer/migrations/20270906208850_alter_nft_metadata_addtl_indexes.sql delete mode 100644 packages/eventindexer/migrations/20270906208851_alter_nft_balances_addtl_indexes.sql delete mode 100644 packages/eventindexer/nft_metadata.go delete mode 100644 packages/eventindexer/pkg/mock/nft_metadata_repository.go delete mode 100644 packages/eventindexer/pkg/repo/nft_metadata.go diff --git a/.github/workflows/eventindexer.yml b/.github/workflows/eventindexer.yml index 4699550398a..b6b42eaefec 100644 --- a/.github/workflows/eventindexer.yml +++ b/.github/workflows/eventindexer.yml @@ -44,6 +44,9 @@ jobs: with: access_token: ${{ github.token }} + - name: Install Git + run: sudo apt-get update && sudo apt-get install -y git + - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: diff --git a/packages/eventindexer/api/api.go b/packages/eventindexer/api/api.go index 8bbe3ae13cf..7e156b55572 100644 --- a/packages/eventindexer/api/api.go +++ b/packages/eventindexer/api/api.go @@ -76,11 +76,6 @@ func InitFromConfig(ctx context.Context, api *API, cfg *Config) error { return err } - nftMetadataRepository, err := repo.NewNFTMetadataRepository(db) - if err != nil { - return err - } - ethClient, err := ethclient.Dial(cfg.RPCUrl) if err != nil { return err @@ -89,7 +84,6 @@ func InitFromConfig(ctx context.Context, api *API, cfg *Config) error { srv, err := http.NewServer(http.NewServerOpts{ EventRepo: eventRepository, NFTBalanceRepo: nftBalanceRepository, - NFTMetadataRepo: nftMetadataRepository, ERC20BalanceRepo: erc20BalanceRepository, ChartRepo: chartRepository, Echo: echo.New(), diff --git a/packages/eventindexer/errors.go b/packages/eventindexer/errors.go index 8a2e833b55d..d85fc6c3d79 100644 --- a/packages/eventindexer/errors.go +++ b/packages/eventindexer/errors.go @@ -12,10 +12,6 @@ var ( "ERR_NO_NFT_BALANCE_REPOSITORY", "NFTBalanceRepository is required", ) - ErrNoNFTMetadataRepository = errors.Validation.NewWithKeyAndDetail( - "ERR_NO_NFT_METADATA_REPOSITORY", - "NFTMetadataRepository is required", - ) ErrNoStatRepository = errors.Validation.NewWithKeyAndDetail( "ERR_NO_STAT_REPOSITORY", "StatRepository is required", diff --git a/packages/eventindexer/indexer/fetch_nft_metadata.go b/packages/eventindexer/indexer/fetch_nft_metadata.go deleted file mode 100644 index b53f558f856..00000000000 --- a/packages/eventindexer/indexer/fetch_nft_metadata.go +++ /dev/null @@ -1,180 +0,0 @@ -package indexer - -import ( - "context" - "encoding/base64" - "encoding/json" - "fmt" - "math/big" - "net/http" - "net/url" - "strings" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/common" - "github.com/pkg/errors" - "github.com/taikoxyz/taiko-mono/packages/eventindexer" - "github.com/taikoxyz/taiko-mono/packages/eventindexer/contracts/erc1155" - "github.com/taikoxyz/taiko-mono/packages/eventindexer/contracts/erc721" -) - -func (i *Indexer) fetchNFTMetadata( - ctx context.Context, contractAddress string, - tokenID *big.Int, - abiJSON string, - methodName string, - chainID *big.Int) (*eventindexer.NFTMetadata, error) { - contractABI, err := abi.JSON(strings.NewReader(abiJSON)) - if err != nil { - return nil, err - } - - contractAddressCommon := common.HexToAddress(contractAddress) - - callData, err := contractABI.Pack(methodName, tokenID) - if err != nil { - return nil, err - } - - msg := ethereum.CallMsg{ - To: &contractAddressCommon, - Data: callData, - } - - result, err := i.ethClient.CallContract(ctx, msg, nil) - if err != nil { - return nil, errors.Wrap(err, "i.ethClient.CallContract") - } - - var tokenURI string - - err = contractABI.UnpackIntoInterface(&tokenURI, methodName, result) - if err != nil { - return nil, errors.Wrap(err, "contractABI.UnpackIntoInterface") - } - - mdURL := resolveMetadataURL(tokenURI) - - if !isValidURL(mdURL) { - return nil, nil - } - - var metadata *eventindexer.NFTMetadata - - //nolint - resp, err := http.Get(mdURL) - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, nil - } - - err = json.NewDecoder(resp.Body).Decode(&metadata) - if err != nil { - return nil, err - } - - if methodName == "tokenURI" { - if err := i.fetchSymbol(ctx, contractABI, metadata, contractAddressCommon); err != nil { - return nil, err - } - } - - metadata.ContractAddress = contractAddress - metadata.TokenID = tokenID.Int64() - metadata.ChainID = chainID.Int64() - - return metadata, nil -} - -// isValidURL checks if the given string is a valid URL -func isValidURL(str string) bool { - u, err := url.Parse(str) - if err != nil || u.Scheme == "" || u.Host == "" { - return false - } - - return true -} - -// isBase64 checks if the given string is a valid base64 encoded string -func isBase64(str string) bool { - if !strings.Contains(str, "base64,") { - return false - } - - parts := strings.Split(str, "base64,") - if len(parts) != 2 { - return false - } - - _, err := base64.StdEncoding.DecodeString(parts[1]) - - return err == nil -} - -func resolveMetadataURL(tokenURI string) string { - if strings.HasPrefix(tokenURI, "ipfs://") { - ipfsHash := strings.TrimPrefix(tokenURI, "ipfs://") - resolvedURL := fmt.Sprintf("https://ipfs.io/ipfs/%s", ipfsHash) - - return resolvedURL - } - - if isBase64(tokenURI) { - parts := strings.Split(tokenURI, "base64,") - - decodedTokenURI, _ := base64.StdEncoding.DecodeString(parts[1]) - - return string(decodedTokenURI) - } - - return tokenURI -} - -func (i *Indexer) fetchSymbol(ctx context.Context, contractABI abi.ABI, metadata *eventindexer.NFTMetadata, contractAddress common.Address) error { - symbolCallData, err := contractABI.Pack("symbol") - if err != nil { - return errors.Wrap(err, "contractABI.Pack") - } - - symbolMsg := ethereum.CallMsg{ - To: &contractAddress, - Data: symbolCallData, - } - - symbolResult, err := i.ethClient.CallContract(ctx, symbolMsg, nil) - if err != nil { - return errors.Wrap(err, "i.ethClient.CallContract(symbolMsg)") - } - - var symbol string - - err = contractABI.UnpackIntoInterface(&symbol, "symbol", symbolResult) - if err != nil { - return errors.Wrap(err, "contractABI.UnpackIntoInterface") - } - - metadata.Symbol = symbol - - return nil -} - -func (i *Indexer) fetchERC721Metadata(ctx context.Context, - contractAddress string, - tokenID *big.Int, - chainID *big.Int) (*eventindexer.NFTMetadata, error) { - return i.fetchNFTMetadata(ctx, contractAddress, tokenID, erc721.ABI, "tokenURI", chainID) -} - -func (i *Indexer) fetchERC1155Metadata(ctx context.Context, - contractAddress string, - tokenID *big.Int, - chainID *big.Int) (*eventindexer.NFTMetadata, error) { - return i.fetchNFTMetadata(ctx, contractAddress, tokenID, erc1155.ABI, "uri", chainID) -} diff --git a/packages/eventindexer/indexer/index_erc20_transfers.go b/packages/eventindexer/indexer/index_erc20_transfers.go index 2d5da493d10..efe091b883c 100644 --- a/packages/eventindexer/indexer/index_erc20_transfers.go +++ b/packages/eventindexer/indexer/index_erc20_transfers.go @@ -57,7 +57,7 @@ func (i *Indexer) indexERC20Transfers( } // isERC20Transfer determines whether a given log is a valid ERC20 transfer event -func (i *Indexer) isERC20Transfer(ctx context.Context, vLog types.Log) bool { +func (i *Indexer) isERC20Transfer(_ context.Context, vLog types.Log) bool { // malformed event if len(vLog.Topics) == 0 { return false diff --git a/packages/eventindexer/indexer/index_nft_transfers.go b/packages/eventindexer/indexer/index_nft_transfers.go index 624f7c46e36..721323a2218 100644 --- a/packages/eventindexer/indexer/index_nft_transfers.go +++ b/packages/eventindexer/indexer/index_nft_transfers.go @@ -77,7 +77,7 @@ func (i *Indexer) isERC1155Transfer(_ context.Context, vLog types.Log) bool { } // isERC721Transfer determines whether a given log is a valid ERC721 transfer event -func (i *Indexer) isERC721Transfer(ctx context.Context, vLog types.Log) bool { +func (i *Indexer) isERC721Transfer(_ context.Context, vLog types.Log) bool { // malformed event if len(vLog.Topics) == 0 { return false @@ -126,54 +126,10 @@ func (i *Indexer) saveERC721Transfer(ctx context.Context, chainID *big.Int, vLog "contractAddress", vLog.Address.Hex(), ) - var pk int = 0 - - // Check if metadata already exists in db, if not fetch and store - metadata, err := i.nftMetadataRepo.GetNFTMetadata(ctx, vLog.Address.Hex(), tokenID, chainID.Int64()) - if err != nil { - return err - } - - if metadata != nil { - pk = metadata.ID - } - - if pk == 0 { - metadata, err = i.fetchERC721Metadata(ctx, vLog.Address.Hex(), vLog.Topics[3].Big(), chainID) - if err != nil { - slog.Warn("error fetching metadata. setting defaults", - "contractAddress", vLog.Address.Hex(), "error", err.Error()) - } - - if metadata == nil { - metadata = &eventindexer.NFTMetadata{ - ChainID: chainID.Int64(), - ContractAddress: vLog.Address.Hex(), - TokenID: tokenID, - Name: "invalid_metadata", - } - - metadata, err = i.nftMetadataRepo.SaveNFTMetadata(ctx, metadata) - if err != nil { - return errors.Wrap(err, "i.nftMetadataRepo.SaveNFTMetadata") - } - } - - pk = metadata.ID - } - - if pk == 0 { - slog.Warn("unable to create or fetch md", "contractAddress", vLog.Address.Hex()) - return nil - } - - slog.Info("metadata pk", "pk", pk) - // increment To address's balance // decrement From address's balance increaseOpts := eventindexer.UpdateNFTBalanceOpts{ ChainID: chainID.Int64(), - NftMetadataId: int64(pk), Address: to, TokenID: tokenID, ContractAddress: vLog.Address.Hex(), @@ -186,7 +142,6 @@ func (i *Indexer) saveERC721Transfer(ctx context.Context, chainID *big.Int, vLog if from != ZeroAddress.Hex() { decreaseOpts = eventindexer.UpdateNFTBalanceOpts{ ChainID: chainID.Int64(), - NftMetadataId: int64(pk), Address: from, TokenID: tokenID, ContractAddress: vLog.Address.Hex(), @@ -195,7 +150,7 @@ func (i *Indexer) saveERC721Transfer(ctx context.Context, chainID *big.Int, vLog } } - _, _, err = i.nftBalanceRepo.IncreaseAndDecreaseBalancesInTx(ctx, increaseOpts, decreaseOpts) + _, _, err := i.nftBalanceRepo.IncreaseAndDecreaseBalancesInTx(ctx, increaseOpts, decreaseOpts) if err != nil { return err } @@ -234,45 +189,8 @@ func (i *Indexer) saveERC1155Transfer(ctx context.Context, chainID *big.Int, vLo return err } - var pk int = 0 - - // Check if metadata already exists in db, if not fetch and store - metadata, err := i.nftMetadataRepo.GetNFTMetadata(ctx, vLog.Address.Hex(), t.Id.Int64(), chainID.Int64()) - if err != nil { - return err - } - - if metadata != nil { - pk = metadata.ID - } - - if pk == 0 { - metadata, err = i.fetchERC1155Metadata(ctx, vLog.Address.Hex(), t.Id, chainID) - if err != nil { - slog.Warn("error fetching metadata. setting defaults", - "contractAddress", vLog.Address.Hex(), "error", err.Error()) - } - - if metadata == nil { - metadata = &eventindexer.NFTMetadata{ - ChainID: chainID.Int64(), - ContractAddress: vLog.Address.Hex(), - TokenID: t.Id.Int64(), - Name: "invalid_metadata", - } - - metadata, err = i.nftMetadataRepo.SaveNFTMetadata(ctx, metadata) - if err != nil { - return errors.Wrap(err, "i.nftMetadataRepo.SaveNFTMetadata") - } - } - - pk = metadata.ID - } - increaseOpts := eventindexer.UpdateNFTBalanceOpts{ ChainID: chainID.Int64(), - NftMetadataId: int64(pk), Address: to, TokenID: t.Id.Int64(), ContractAddress: vLog.Address.Hex(), @@ -285,7 +203,6 @@ func (i *Indexer) saveERC1155Transfer(ctx context.Context, chainID *big.Int, vLo // decrement From address's balance decreaseOpts = eventindexer.UpdateNFTBalanceOpts{ ChainID: chainID.Int64(), - NftMetadataId: int64(pk), Address: from, TokenID: t.Id.Int64(), ContractAddress: vLog.Address.Hex(), @@ -315,44 +232,8 @@ func (i *Indexer) saveERC1155Transfer(ctx context.Context, chainID *big.Int, vLo } for idx, id := range t.Ids { - var pk int = 0 - - // Check if metadata already exists in db, if not fetch and store - metadata, err := i.nftMetadataRepo.GetNFTMetadata(ctx, vLog.Address.Hex(), id.Int64(), chainID.Int64()) - if err != nil { - return err - } - - if metadata != nil { - pk = metadata.ID - } - - if pk == 0 { - metadata, err = i.fetchERC1155Metadata(ctx, vLog.Address.Hex(), id, chainID) - if err != nil { - return err - } - - if metadata == nil { - metadata = &eventindexer.NFTMetadata{ - ChainID: chainID.Int64(), - ContractAddress: vLog.Address.Hex(), - TokenID: id.Int64(), - Name: "invalid_metadata", - } - - metadata, err = i.nftMetadataRepo.SaveNFTMetadata(ctx, metadata) - if err != nil { - return errors.Wrap(err, "i.nftMetadataRepo.SaveNFTMetadata") - } - } - - pk = metadata.ID - } - increaseOpts := eventindexer.UpdateNFTBalanceOpts{ ChainID: chainID.Int64(), - NftMetadataId: int64(pk), Address: to, TokenID: id.Int64(), ContractAddress: vLog.Address.Hex(), @@ -365,7 +246,6 @@ func (i *Indexer) saveERC1155Transfer(ctx context.Context, chainID *big.Int, vLo // decrement From address's balance decreaseOpts = eventindexer.UpdateNFTBalanceOpts{ ChainID: chainID.Int64(), - NftMetadataId: int64(pk), Address: from, TokenID: id.Int64(), ContractAddress: vLog.Address.Hex(), diff --git a/packages/eventindexer/indexer/indexer.go b/packages/eventindexer/indexer/indexer.go index 836f4bce3bb..6093865f3bf 100644 --- a/packages/eventindexer/indexer/indexer.go +++ b/packages/eventindexer/indexer/indexer.go @@ -41,7 +41,6 @@ type Indexer struct { accountRepo eventindexer.AccountRepository eventRepo eventindexer.EventRepository nftBalanceRepo eventindexer.NFTBalanceRepository - nftMetadataRepo eventindexer.NFTMetadataRepository erc20BalanceRepo eventindexer.ERC20BalanceRepository txRepo eventindexer.TransactionRepository @@ -145,11 +144,6 @@ func InitFromConfig(ctx context.Context, i *Indexer, cfg *Config) error { return err } - nftMetadataRepository, err := repo.NewNFTMetadataRepository(db) - if err != nil { - return err - } - txRepository, err := repo.NewTransactionRepository(db) if err != nil { return err @@ -193,7 +187,6 @@ func InitFromConfig(ctx context.Context, i *Indexer, cfg *Config) error { i.eventRepo = eventRepository i.nftBalanceRepo = nftBalanceRepository i.erc20BalanceRepo = erc20BalanceRepository - i.nftMetadataRepo = nftMetadataRepository i.txRepo = txRepository i.srcChainID = chainID.Uint64() diff --git a/packages/eventindexer/migrations/20270906208845_create_nft_metadata_table.sql b/packages/eventindexer/migrations/20270906208845_create_nft_metadata_table.sql deleted file mode 100644 index d99e396f797..00000000000 --- a/packages/eventindexer/migrations/20270906208845_create_nft_metadata_table.sql +++ /dev/null @@ -1,23 +0,0 @@ --- +goose Up --- +goose StatementBegin -CREATE TABLE IF NOT EXISTS nft_metadata ( - id int NOT NULL PRIMARY KEY AUTO_INCREMENT, - chain_id int NOT NULL, - contract_address VARCHAR(42) NOT NULL, - token_id DECIMAL(65, 0) NOT NULL, - name VARCHAR(255) DEFAULT NULL, - description TEXT DEFAULT NULL, - symbol VARCHAR(10) DEFAULT NULL, - attributes JSON DEFAULT NULL, - image_url TEXT DEFAULT NULL, - image_data TEXT DEFAULT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - UNIQUE (contract_address, token_id) -); --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -DROP TABLE nft_metadata; --- +goose StatementEnd diff --git a/packages/eventindexer/migrations/20270906208846_alter_nft_metadata_add_indexes.sql b/packages/eventindexer/migrations/20270906208846_alter_nft_metadata_add_indexes.sql deleted file mode 100644 index 6c95bbe7e57..00000000000 --- a/packages/eventindexer/migrations/20270906208846_alter_nft_metadata_add_indexes.sql +++ /dev/null @@ -1,10 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE `nft_metadata` ADD INDEX `nft_metadata_contract_address_token_id_index` (`contract_address`, `token_id`); - --- +goose StatementEnd --- +goose Down --- +goose StatementBegin -ALTER TABLE nft_metadata - DROP INDEX nft_metadata_contract_address_token_id_index --- +goose StatementEnd diff --git a/packages/eventindexer/migrations/20270906208847_alter_nft_metadata_table_update_symbol_length.sql b/packages/eventindexer/migrations/20270906208847_alter_nft_metadata_table_update_symbol_length.sql deleted file mode 100644 index 412f64e63b9..00000000000 --- a/packages/eventindexer/migrations/20270906208847_alter_nft_metadata_table_update_symbol_length.sql +++ /dev/null @@ -1,9 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE nft_metadata MODIFY symbol VARCHAR(42) DEFAULT NULL; --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE nft_metadata MODIFY symbol VARCHAR(10) DEFAULT NULL; --- +goose StatementEnd \ No newline at end of file diff --git a/packages/eventindexer/migrations/20270906208848_alter_nft_balances_add_metadata_column.sql b/packages/eventindexer/migrations/20270906208848_alter_nft_balances_add_metadata_column.sql deleted file mode 100644 index 43733bb47b7..00000000000 --- a/packages/eventindexer/migrations/20270906208848_alter_nft_balances_add_metadata_column.sql +++ /dev/null @@ -1,14 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE nft_balances - ADD COLUMN nft_metadata_id INT NULL DEFAULT 0, - ADD CONSTRAINT fk_nft_balances_nft_metadata - FOREIGN KEY (nft_metadata_id) REFERENCES nft_metadata(id); --- +goose StatementEnd - --- +goose Down --- +goose StatementBegin -ALTER TABLE nft_balances - DROP FOREIGN KEY fk_nft_balances_nft_metadata, - DROP COLUMN nft_metadata_id; --- +goose StatementEnd diff --git a/packages/eventindexer/migrations/20270906208850_alter_nft_metadata_addtl_indexes.sql b/packages/eventindexer/migrations/20270906208850_alter_nft_metadata_addtl_indexes.sql deleted file mode 100644 index 8e9e3bfd923..00000000000 --- a/packages/eventindexer/migrations/20270906208850_alter_nft_metadata_addtl_indexes.sql +++ /dev/null @@ -1,12 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE `nft_metadata` ADD INDEX `nft_metadata_contract_address_token_id_chain_id_index` (`contract_address`, `token_id`, `chain_id`), -ADD INDEX `nft_metadata_contract_address_index` (`contract_address`) -; - --- +goose StatementEnd --- +goose Down --- +goose StatementBegin -ALTER TABLE nft_metadata DROP INDEX nft_metadata_contract_address_token_id_chain_id_index, - DROP INDEX nft_metadata_contract_address; --- +goose StatementEnd diff --git a/packages/eventindexer/migrations/20270906208851_alter_nft_balances_addtl_indexes.sql b/packages/eventindexer/migrations/20270906208851_alter_nft_balances_addtl_indexes.sql deleted file mode 100644 index e641c74a96b..00000000000 --- a/packages/eventindexer/migrations/20270906208851_alter_nft_balances_addtl_indexes.sql +++ /dev/null @@ -1,10 +0,0 @@ --- +goose Up --- +goose StatementBegin -ALTER TABLE `nft_balances` ADD INDEX `nft_balances_address_chain_id_amount_index` (`address`, `chain_id`, `amount`); - --- +goose StatementEnd --- +goose Down --- +goose StatementBegin -ALTER TABLE nft_balances - DROP INDEX nft_balances_address_chain_id_amount_index --- +goose StatementEnd diff --git a/packages/eventindexer/nft_balance.go b/packages/eventindexer/nft_balance.go index f918ada9743..1497b32d6e3 100644 --- a/packages/eventindexer/nft_balance.go +++ b/packages/eventindexer/nft_balance.go @@ -10,19 +10,16 @@ import ( // NFTBalance represents a single contractAddress/tokenId pairing for a given holder // address type NFTBalance struct { - ID int `json:"id"` - NftMetadataId int64 `json:"nftMetadataId"` - ChainID int64 `json:"chainID"` - Address string `json:"address"` - Amount int64 `json:"amount"` - TokenID int64 `json:"tokenID"` - ContractAddress string `json:"contractAddress"` - ContractType string `json:"contractType"` - Metadata *NFTMetadata `json:"metadata" gorm:"foreignKey:NftMetadataId"` + ID int `json:"id"` + ChainID int64 `json:"chainID"` + Address string `json:"address"` + Amount int64 `json:"amount"` + TokenID int64 `json:"tokenID"` + ContractAddress string `json:"contractAddress"` + ContractType string `json:"contractType"` } type UpdateNFTBalanceOpts struct { - NftMetadataId int64 ChainID int64 Address string TokenID int64 diff --git a/packages/eventindexer/nft_metadata.go b/packages/eventindexer/nft_metadata.go deleted file mode 100644 index a21dc86602a..00000000000 --- a/packages/eventindexer/nft_metadata.go +++ /dev/null @@ -1,102 +0,0 @@ -package eventindexer - -import ( - "context" - "database/sql/driver" - "encoding/json" - "errors" - "fmt" - "net/http" - "strconv" - - "github.com/morkid/paginate" -) - -type Attribute struct { - TraitType string `json:"trait_type"` - Value string `json:"value"` -} - -type Attributes []Attribute - -type NFTMetadata struct { - ID int `json:"id"` - ChainID int64 `json:"chain_id"` - ContractAddress string `json:"contract_address"` - TokenID int64 `json:"token_id"` - Name string `json:"name,omitempty"` - Description string `json:"description,omitempty"` - Symbol string `json:"symbol,omitempty"` - Attributes Attributes `json:"attributes,omitempty"` - ImageURL string `json:"image_url,omitempty"` - ImageData string `json:"image_data,omitempty"` -} - -type NFTMetadataRepository interface { - SaveNFTMetadata(ctx context.Context, metadata *NFTMetadata) (*NFTMetadata, error) - GetNFTMetadata(ctx context.Context, contractAddress string, tokenID int64, chainID int64) (*NFTMetadata, error) - FindByContractAddress(ctx context.Context, req *http.Request, contractAddress string) (paginate.Page, error) -} - -func (n *NFTMetadata) UnmarshalJSON(data []byte) error { - type Alias NFTMetadata - - aux := &struct { - ImageURL string `json:"image_url"` - Image string `json:"image"` - *Alias - }{ - Alias: (*Alias)(n), - } - - if err := json.Unmarshal(data, &aux); err != nil { - return fmt.Errorf("failed to unmarshal JSON: %v", err) - } - - if aux.ImageURL != "" { - n.ImageURL = aux.ImageURL - } else { - n.ImageURL = aux.Image - } - - return nil -} - -func (a Attributes) Value() (driver.Value, error) { - return json.Marshal(a) -} - -func (a *Attributes) Scan(value interface{}) error { - bytes, ok := value.([]byte) - if !ok { - return errors.New("type assertion to []byte failed") - } - - return json.Unmarshal(bytes, a) -} - -func (a *Attribute) UnmarshalJSON(data []byte) error { - type Alias Attribute - - aux := &struct { - Value interface{} `json:"value"` - *Alias - }{ - Alias: (*Alias)(a), - } - - if err := json.Unmarshal(data, &aux); err != nil { - return err - } - - switch v := aux.Value.(type) { - case string: - a.Value = v - case float64: - a.Value = strconv.FormatFloat(v, 'f', -1, 64) - default: - return fmt.Errorf("unexpected type for value field: %T", aux.Value) - } - - return nil -} diff --git a/packages/eventindexer/pkg/http/get_by_address_and_event_test.go b/packages/eventindexer/pkg/http/get_by_address_and_event_test.go index 8923252bd60..22d9ce8742b 100644 --- a/packages/eventindexer/pkg/http/get_by_address_and_event_test.go +++ b/packages/eventindexer/pkg/http/get_by_address_and_event_test.go @@ -16,7 +16,7 @@ import ( ) func Test_GetByAddressAndEvent(t *testing.T) { - srv := newTestServer("") + srv := newTestServer() _, err := srv.eventRepo.Save(context.Background(), eventindexer.SaveEventOpts{ Name: "name", diff --git a/packages/eventindexer/pkg/http/get_count_by_address_and_event_test.go b/packages/eventindexer/pkg/http/get_count_by_address_and_event_test.go index 6a3d6819c62..23df67f30dc 100644 --- a/packages/eventindexer/pkg/http/get_count_by_address_and_event_test.go +++ b/packages/eventindexer/pkg/http/get_count_by_address_and_event_test.go @@ -16,7 +16,7 @@ import ( ) func Test_GetCountByAddressAndEvent(t *testing.T) { - srv := newTestServer("") + srv := newTestServer() _, err := srv.eventRepo.Save(context.Background(), eventindexer.SaveEventOpts{ Name: "name", diff --git a/packages/eventindexer/pkg/http/get_nft_balances_by_address_and_chain_id.go b/packages/eventindexer/pkg/http/get_nft_balances_by_address_and_chain_id.go index bef4f8ae00c..45a017bd24c 100644 --- a/packages/eventindexer/pkg/http/get_nft_balances_by_address_and_chain_id.go +++ b/packages/eventindexer/pkg/http/get_nft_balances_by_address_and_chain_id.go @@ -5,18 +5,8 @@ import ( "github.com/cyberhorsey/webutils" "github.com/labstack/echo/v4" - "github.com/taikoxyz/taiko-mono/packages/eventindexer" ) -type NFTBalanceWithMetadata struct { - Balance eventindexer.NFTBalance `json:"balance"` - Metadata *eventindexer.NFTMetadata `json:"metadata"` -} - -type NFTBalancesResponse struct { - Balances []NFTBalanceWithMetadata `json:"balances"` -} - // GetNFTBalancesByAddressAndChainID // // returns nft balances by address and chain ID @@ -40,21 +30,5 @@ func (srv *Server) GetNFTBalancesByAddressAndChainID(c echo.Context) error { return webutils.LogAndRenderErrors(c, http.StatusUnprocessableEntity, err) } - for i := range *page.Items.(*[]eventindexer.NFTBalance) { - v := &(*page.Items.(*[]eventindexer.NFTBalance))[i] - - metadata, err := srv.nftMetadataRepo.GetNFTMetadata( - c.Request().Context(), - v.ContractAddress, - v.TokenID, - v.ChainID, - ) - if err != nil { - return webutils.LogAndRenderErrors(c, http.StatusUnprocessableEntity, err) - } - - v.Metadata = metadata - } - return c.JSON(http.StatusOK, page) } diff --git a/packages/eventindexer/pkg/http/get_unique_proposers_test.go b/packages/eventindexer/pkg/http/get_unique_proposers_test.go index bdfcfcb6dcc..fa2e867ae67 100644 --- a/packages/eventindexer/pkg/http/get_unique_proposers_test.go +++ b/packages/eventindexer/pkg/http/get_unique_proposers_test.go @@ -15,7 +15,7 @@ import ( ) func Test_GetUniqueProposers(t *testing.T) { - srv := newTestServer("") + srv := newTestServer() _, err := srv.eventRepo.Save(context.Background(), eventindexer.SaveEventOpts{ Name: "name", diff --git a/packages/eventindexer/pkg/http/get_unique_provers_test.go b/packages/eventindexer/pkg/http/get_unique_provers_test.go index a8a586fe1a2..72353cc2f86 100644 --- a/packages/eventindexer/pkg/http/get_unique_provers_test.go +++ b/packages/eventindexer/pkg/http/get_unique_provers_test.go @@ -15,7 +15,7 @@ import ( ) func Test_GetUniqueProvers(t *testing.T) { - srv := newTestServer("") + srv := newTestServer() _, err := srv.eventRepo.Save(context.Background(), eventindexer.SaveEventOpts{ Name: "name", diff --git a/packages/eventindexer/pkg/http/server.go b/packages/eventindexer/pkg/http/server.go index 5cad0de403a..4626bf590d4 100644 --- a/packages/eventindexer/pkg/http/server.go +++ b/packages/eventindexer/pkg/http/server.go @@ -31,7 +31,6 @@ type Server struct { echo *echo.Echo eventRepo eventindexer.EventRepository nftBalanceRepo eventindexer.NFTBalanceRepository - nftMetadataRepo eventindexer.NFTMetadataRepository erc20BalanceRepo eventindexer.ERC20BalanceRepository chartRepo eventindexer.ChartRepository cache *cache.Cache @@ -41,7 +40,6 @@ type NewServerOpts struct { Echo *echo.Echo EventRepo eventindexer.EventRepository NFTBalanceRepo eventindexer.NFTBalanceRepository - NFTMetadataRepo eventindexer.NFTMetadataRepository ERC20BalanceRepo eventindexer.ERC20BalanceRepository ChartRepo eventindexer.ChartRepository EthClient *ethclient.Client @@ -61,10 +59,6 @@ func (opts NewServerOpts) Validate() error { return eventindexer.ErrNoNFTBalanceRepository } - if opts.NFTMetadataRepo == nil { - return eventindexer.ErrNoNFTMetadataRepository - } - return nil } @@ -79,7 +73,6 @@ func NewServer(opts NewServerOpts) (*Server, error) { echo: opts.Echo, eventRepo: opts.EventRepo, nftBalanceRepo: opts.NFTBalanceRepo, - nftMetadataRepo: opts.NFTMetadataRepo, erc20BalanceRepo: opts.ERC20BalanceRepo, chartRepo: opts.ChartRepo, cache: cache, diff --git a/packages/eventindexer/pkg/http/server_test.go b/packages/eventindexer/pkg/http/server_test.go index 5eeff772750..ea28c836bd4 100644 --- a/packages/eventindexer/pkg/http/server_test.go +++ b/packages/eventindexer/pkg/http/server_test.go @@ -16,7 +16,7 @@ import ( "github.com/taikoxyz/taiko-mono/packages/eventindexer/pkg/repo" ) -func newTestServer(url string) *Server { +func newTestServer() *Server { _ = godotenv.Load("../.test.env") srv := &Server{ @@ -24,7 +24,6 @@ func newTestServer(url string) *Server { echo: echo.New(), eventRepo: mock.NewEventRepository(), nftBalanceRepo: mock.NewNFTBalanceRepository(), - nftMetadataRepo: mock.NewNFTMetadataRepository(), erc20BalanceRepo: mock.NewERC20BalanceRepository(), } @@ -47,7 +46,6 @@ func Test_NewServer(t *testing.T) { EventRepo: &repo.EventRepository{}, CorsOrigins: make([]string, 0), NFTBalanceRepo: &repo.NFTBalanceRepository{}, - NFTMetadataRepo: &repo.NFTMetadataRepository{}, ERC20BalanceRepo: &repo.ERC20BalanceRepository{}, }, nil, @@ -61,46 +59,15 @@ func Test_NewServer(t *testing.T) { }, eventindexer.ErrNoNFTBalanceRepository, }, - { - "noNftMetadataRepo", - NewServerOpts{ - Echo: echo.New(), - EventRepo: &repo.EventRepository{}, - NFTBalanceRepo: &repo.NFTBalanceRepository{}, - CorsOrigins: make([]string, 0), - }, - eventindexer.ErrNoNFTMetadataRepository, - }, { "noEventRepo", NewServerOpts{ Echo: echo.New(), - EventRepo: &repo.EventRepository{}, - NFTBalanceRepo: &repo.NFTBalanceRepository{}, CorsOrigins: make([]string, 0), - }, - eventindexer.ErrNoNFTMetadataRepository, - }, - { - "noEventRepo", - NewServerOpts{ - Echo: echo.New(), - CorsOrigins: make([]string, 0), - NFTBalanceRepo: &repo.NFTBalanceRepository{}, - NFTMetadataRepo: &repo.NFTMetadataRepository{}, + NFTBalanceRepo: &repo.NFTBalanceRepository{}, }, eventindexer.ErrNoEventRepository, }, - { - "noCorsOrigins", - NewServerOpts{ - Echo: echo.New(), - EventRepo: &repo.EventRepository{}, - NFTBalanceRepo: &repo.NFTBalanceRepository{}, - NFTMetadataRepo: &repo.NFTMetadataRepository{}, - }, - eventindexer.ErrNoCORSOrigins, - }, { "noHttpFramework", NewServerOpts{ @@ -119,7 +86,7 @@ func Test_NewServer(t *testing.T) { } func Test_Health(t *testing.T) { - srv := newTestServer("") + srv := newTestServer() req, _ := http.NewRequest(echo.GET, "/healthz", nil) rec := httptest.NewRecorder() @@ -132,7 +99,7 @@ func Test_Health(t *testing.T) { } func Test_Root(t *testing.T) { - srv := newTestServer("") + srv := newTestServer() req, _ := http.NewRequest(echo.GET, "/", nil) rec := httptest.NewRecorder() @@ -145,7 +112,7 @@ func Test_Root(t *testing.T) { } func Test_StartShutdown(t *testing.T) { - srv := newTestServer("") + srv := newTestServer() go func() { _ = srv.Start(":3928") diff --git a/packages/eventindexer/pkg/mock/nft_metadata_repository.go b/packages/eventindexer/pkg/mock/nft_metadata_repository.go deleted file mode 100644 index 4db7a44e0b5..00000000000 --- a/packages/eventindexer/pkg/mock/nft_metadata_repository.go +++ /dev/null @@ -1,57 +0,0 @@ -package mock - -import ( - "context" - "net/http" - - "github.com/morkid/paginate" - "github.com/taikoxyz/taiko-mono/packages/eventindexer" -) - -type NFTMetadataRepository struct { - nftMetadata []*eventindexer.NFTMetadata -} - -func NewNFTMetadataRepository() *NFTMetadataRepository { - return &NFTMetadataRepository{} -} - -func (r *NFTMetadataRepository) FindByContractAddress( - ctx context.Context, - req *http.Request, - contractAddress string) (paginate.Page, error) { - var metadata []*eventindexer.NFTMetadata - - for _, b := range r.nftMetadata { - if b.ContractAddress == contractAddress { - metadata = append(metadata, b) - } - } - - return paginate.Page{ - Items: metadata, - }, nil -} - -func (r *NFTMetadataRepository) GetNFTMetadata( - ctx context.Context, - contractAddress string, - tokenID int64, - chainID int64, -) (*eventindexer.NFTMetadata, error) { - for _, metadata := range r.nftMetadata { - if metadata.ContractAddress == contractAddress && metadata.TokenID == tokenID { - return metadata, nil - } - } - - return nil, nil -} - -func (r *NFTMetadataRepository) SaveNFTMetadata( - ctx context.Context, - metadata *eventindexer.NFTMetadata) (*eventindexer.NFTMetadata, error) { - r.nftMetadata = append(r.nftMetadata, metadata) - - return metadata, nil -} diff --git a/packages/eventindexer/pkg/repo/nft_balance.go b/packages/eventindexer/pkg/repo/nft_balance.go index 318a8a28b89..3c83174035c 100644 --- a/packages/eventindexer/pkg/repo/nft_balance.go +++ b/packages/eventindexer/pkg/repo/nft_balance.go @@ -36,14 +36,13 @@ func (r *NFTBalanceRepository) increaseBalanceInDB( ) (*eventindexer.NFTBalance, error) { b := &eventindexer.NFTBalance{ ContractAddress: opts.ContractAddress, - NftMetadataId: opts.NftMetadataId, TokenID: opts.TokenID, Address: opts.Address, ContractType: opts.ContractType, ChainID: opts.ChainID, } - err := db. + err := db.WithContext(ctx). Where("contract_address = ?", opts.ContractAddress). Where("token_id = ?", opts.TokenID). Where("address = ?", opts.Address). @@ -74,14 +73,13 @@ func (r *NFTBalanceRepository) decreaseBalanceInDB( ) (*eventindexer.NFTBalance, error) { b := &eventindexer.NFTBalance{ ContractAddress: opts.ContractAddress, - NftMetadataId: opts.NftMetadataId, TokenID: opts.TokenID, Address: opts.Address, ContractType: opts.ContractType, ChainID: opts.ChainID, } - err := db. + err := db.WithContext(ctx). Where("contract_address = ?", opts.ContractAddress). Where("token_id = ?", opts.TokenID). Where("address = ?", opts.Address). diff --git a/packages/eventindexer/pkg/repo/nft_balance_test.go b/packages/eventindexer/pkg/repo/nft_balance_test.go index c713e6d4f36..89d994a7cbd 100644 --- a/packages/eventindexer/pkg/repo/nft_balance_test.go +++ b/packages/eventindexer/pkg/repo/nft_balance_test.go @@ -49,22 +49,9 @@ func TestIntegration_NFTBalance_Increase_And_Decrease(t *testing.T) { nftBalanceRepo, err := NewNFTBalanceRepository(db) assert.Equal(t, nil, err) - nftMetadataRepo, err := NewNFTMetadataRepository(db) - assert.Equal(t, nil, err) - - md, err := nftMetadataRepo.SaveNFTMetadata(context.Background(), &eventindexer.NFTMetadata{ - ChainID: 1, - ContractAddress: "0x123", - TokenID: 1, - Name: "test", - }) - assert.Equal(t, nil, err) - assert.NotNil(t, md) - bal1, _, err := nftBalanceRepo.IncreaseAndDecreaseBalancesInTx(context.Background(), eventindexer.UpdateNFTBalanceOpts{ ChainID: 1, - NftMetadataId: int64(md.ID), Address: "0x123", TokenID: 1, ContractAddress: "0x123", @@ -77,7 +64,6 @@ func TestIntegration_NFTBalance_Increase_And_Decrease(t *testing.T) { bal2, _, err := nftBalanceRepo.IncreaseAndDecreaseBalancesInTx(context.Background(), eventindexer.UpdateNFTBalanceOpts{ ChainID: 1, - NftMetadataId: int64(md.ID), Address: "0x123", TokenID: 1, ContractAddress: "0x123456", @@ -97,7 +83,6 @@ func TestIntegration_NFTBalance_Increase_And_Decrease(t *testing.T) { "success", eventindexer.UpdateNFTBalanceOpts{ ChainID: 1, - NftMetadataId: int64(md.ID), Address: "0x123", TokenID: 1, ContractAddress: "0x123456789", @@ -106,7 +91,6 @@ func TestIntegration_NFTBalance_Increase_And_Decrease(t *testing.T) { }, eventindexer.UpdateNFTBalanceOpts{ ChainID: 1, - NftMetadataId: int64(md.ID), Address: "0x123", TokenID: 1, ContractAddress: "0x123", @@ -119,7 +103,6 @@ func TestIntegration_NFTBalance_Increase_And_Decrease(t *testing.T) { "one left", eventindexer.UpdateNFTBalanceOpts{ ChainID: 1, - NftMetadataId: int64(md.ID), Address: "0x123", TokenID: 1, ContractAddress: "0x123456789", @@ -128,7 +111,6 @@ func TestIntegration_NFTBalance_Increase_And_Decrease(t *testing.T) { }, eventindexer.UpdateNFTBalanceOpts{ ChainID: 1, - NftMetadataId: int64(md.ID), Address: "0x123", TokenID: 1, ContractAddress: "0x123456", diff --git a/packages/eventindexer/pkg/repo/nft_metadata.go b/packages/eventindexer/pkg/repo/nft_metadata.go deleted file mode 100644 index 85168fe9c7c..00000000000 --- a/packages/eventindexer/pkg/repo/nft_metadata.go +++ /dev/null @@ -1,93 +0,0 @@ -package repo - -import ( - "context" - "net/http" - - "github.com/morkid/paginate" - "github.com/pkg/errors" - "gorm.io/gorm" - - "github.com/taikoxyz/taiko-mono/packages/eventindexer" - "github.com/taikoxyz/taiko-mono/packages/eventindexer/pkg/db" -) - -type NFTMetadataRepository struct { - db db.DB -} - -func NewNFTMetadataRepository(dbHandler db.DB) (*NFTMetadataRepository, error) { - if dbHandler == nil { - return nil, db.ErrNoDB - } - - return &NFTMetadataRepository{ - db: dbHandler, - }, nil -} - -func (r *NFTMetadataRepository) SaveNFTMetadata( - ctx context.Context, - metadata *eventindexer.NFTMetadata, -) (*eventindexer.NFTMetadata, error) { - existingMetadata, err := r.GetNFTMetadata(ctx, metadata.ContractAddress, metadata.TokenID, metadata.ChainID) - if err != nil { - return nil, errors.Wrap(err, "failed to check existing metadata") - } - - if existingMetadata != nil { - return existingMetadata, nil - } - - err = r.db.GormDB().Save(metadata).Error - if err != nil { - return nil, errors.Wrap(err, "r.db.Save") - } - - return metadata, nil -} - -func (r *NFTMetadataRepository) GetNFTMetadata( - ctx context.Context, - contractAddress string, - tokenID int64, - chainID int64, -) (*eventindexer.NFTMetadata, error) { - metadata := &eventindexer.NFTMetadata{} - - err := r.db.GormDB(). - Where("contract_address = ?", contractAddress). - Where("token_id = ?", tokenID). - Where("chain_id = ?", chainID). - First(metadata). - Error - - if err != nil { - if err == gorm.ErrRecordNotFound { - return nil, nil - } - - return nil, errors.Wrap(err, "r.db.First") - } - - return metadata, nil -} - -func (r *NFTMetadataRepository) FindByContractAddress( - ctx context.Context, - req *http.Request, - contractAddress string, -) (paginate.Page, error) { - pg := paginate.New(&paginate.Config{ - DefaultSize: 100, - }) - - q := r.db.GormDB(). - Raw("SELECT * FROM nft_metadata WHERE contract_address = ?", contractAddress) - - reqCtx := pg.With(q) - - page := reqCtx.Request(req).Response(&[]eventindexer.NFTMetadata{}) - - return page, nil -} diff --git a/packages/relayer/pkg/http/server_test.go b/packages/relayer/pkg/http/server_test.go index 4d13f7807ed..061861158b9 100644 --- a/packages/relayer/pkg/http/server_test.go +++ b/packages/relayer/pkg/http/server_test.go @@ -75,16 +75,6 @@ func Test_NewServer(t *testing.T) { }, relayer.ErrNoEventRepository, }, - { - "noCorsOrigins", - NewServerOpts{ - Echo: echo.New(), - EventRepo: &repo.EventRepository{}, - SrcEthClient: &mock.EthClient{}, - DestEthClient: &mock.EthClient{}, - }, - relayer.ErrNoCORSOrigins, - }, { "noHttpFramework", NewServerOpts{