Problem: get tx receipt api is too slow for indexing service (#432)

* Problem: get tx receipt api is too slow for indexing service

Closes: #431
Solution:
- add extension api: cronos_getTransactionReceiptsByBlock
- add integration test

* cherry-pick to crypto-org-chain/ethermint fork

* support latest block identifier

* Update x/cronos/rpc/api.go

app/app.go

@@ -123,6 +123,9 @@ import (
 	// Force-load the tracer engines to trigger registration
 	_ "github.com/ethereum/go-ethereum/eth/tracers/js"
 	_ "github.com/ethereum/go-ethereum/eth/tracers/native"
+
+	// force register the extension json-rpc.
+	_ "github.com/crypto-org-chain/cronos/x/cronos/rpc"
 )
 
 const (

go.mod

@@ -164,7 +164,7 @@ replace (
 	// See https://github.com/tecbot/gorocksdb/pull/216
 	github.com/tecbot/gorocksdb => github.com/cosmos/gorocksdb v1.1.1
 
-	github.com/tharsis/ethermint => github.com/crypto-org-chain/ethermint v0.10.0-alpha1-cronos-8
+	github.com/tharsis/ethermint => github.com/crypto-org-chain/ethermint v0.10.0-alpha1-cronos-8.0.20220420023806-5b624f76d0fc
 
 	google.golang.org/grpc => google.golang.org/grpc v1.33.2
 )

go.sum

@@ -246,8 +246,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsr
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/crypto-org-chain/ethermint v0.10.0-alpha1-cronos-8 h1:+1FLhY/C+mKUPqKNqtq81tCIorCFcqLZZflHebRDnIc=
-github.com/crypto-org-chain/ethermint v0.10.0-alpha1-cronos-8/go.mod h1:3CXBYpzlUtEn6OCr1WFw/951MBEeojZW3hcqPCv5ktw=
+github.com/crypto-org-chain/ethermint v0.10.0-alpha1-cronos-8.0.20220420023806-5b624f76d0fc h1:z0LzHZt238Y9sOsQvVI1PciUG7qnWrIM62vRRplJF1g=
+github.com/crypto-org-chain/ethermint v0.10.0-alpha1-cronos-8.0.20220420023806-5b624f76d0fc/go.mod h1:3CXBYpzlUtEn6OCr1WFw/951MBEeojZW3hcqPCv5ktw=
 github.com/crypto-org-chain/ibc-go/v2 v2.2.0-hooks2 h1:elj+Tb/3O9GA3pv62zkc1B0P8hl1WHmF6vF8PInEJm4=
 github.com/crypto-org-chain/ibc-go/v2 v2.2.0-hooks2/go.mod h1:rAHRlBcRiHPP/JszN+08SJx3pegww9bcVncIb9QLx7I=
 github.com/crypto-org-chain/keyring v1.1.6-fixes h1:AUFSu56NY6XobY6XfRoDx6v3loiOrHK5MNUm32GEjwA=

gomod2nix.toml

@@ -3668,13 +3668,13 @@
     sha256 = "1sgjf2vaq554ybc0cwkzn17cz2ibzph2rq0dgaw21c2hym09437x"
 
 ["github.com/tharsis/ethermint"]
-  sumVersion = "v0.10.0-alpha1-cronos-8"
+  sumVersion = "v0.10.0-alpha1-cronos-8.0.20220420023806-5b624f76d0fc"
   vendorPath = "github.com/crypto-org-chain/ethermint"
   ["github.com/tharsis/ethermint".fetch]
     type = "git"
     url = "https://github.com/crypto-org-chain/ethermint"
-    rev = "f4db90bb0575d2ffbe4e2f141c75b7cd638354a9"
-    sha256 = "15gv5hkcf5958rp69m49nxx7kagwpgicn1ry6b6mf23skwxys091"
+    rev = "5b624f76d0fc4d7fbd2bcc84135505de4bf62a21"
+    sha256 = "0mkslhsar47ayb3m1rf70ca13r6qhfd2xzdrnl16gfkpwc6p9pk5"
 
 ["github.com/tidwall/gjson"]
   sumVersion = "v1.6.7"

integration_tests/cosmoscli.py

@@ -505,6 +505,12 @@ def broadcast_tx(self, tx_file, **kwargs):
             self.raw("tx", "broadcast", tx_file, node=self.node_rpc, **kwargs)
         )
 
+    def broadcast_tx_json(self, tx, **kwargs):
+        with tempfile.NamedTemporaryFile("w") as fp:
+            json.dump(tx, fp)
+            fp.flush()
+            return self.broadcast_tx(fp.name)
+
     def unjail(self, addr):
         return json.loads(
             self.raw(

integration_tests/test_basic.py

@@ -1,6 +1,5 @@
 import concurrent.futures
 import json
-import tempfile
 import time
 from pathlib import Path
 
@@ -17,9 +16,11 @@
     KEYS,
     Greeter,
     RevertTestContract,
+    build_batch_tx,
     contract_address,
     contract_path,
     deploy_contract,
+    get_receipts_by_block,
     modify_command_in_supervisor_config,
     send_transaction,
     sign_transaction,
@@ -464,6 +465,7 @@ def test_suicide(cluster):
 def test_batch_tx(cronos):
     "send multiple eth txs in single cosmos tx"
     w3 = cronos.w3
+    cli = cronos.cosmos_cli()
     sender = ADDRS["validator"]
     recipient = ADDRS["community"]
     nonce = w3.eth.get_transaction_count(sender)
@@ -480,51 +482,13 @@ def test_batch_tx(cronos):
         {"from": sender, "nonce": nonce + 2, "gas": 200000}
     )
 
-    signed_txs = [
-        sign_transaction(w3, deploy_tx, KEYS["validator"]),
-        sign_transaction(w3, transfer_tx1, KEYS["validator"]),
-        sign_transaction(w3, transfer_tx2, KEYS["validator"]),
-    ]
-    tmp_txs = [
-        cronos.cosmos_cli().build_evm_tx(signed.rawTransaction.hex())
-        for signed in signed_txs
-    ]
-
-    msgs = [tx["body"]["messages"][0] for tx in tmp_txs]
-    fee = sum(int(tx["auth_info"]["fee"]["amount"][0]["amount"]) for tx in tmp_txs)
-    gas_limit = sum(int(tx["auth_info"]["fee"]["gas_limit"]) for tx in tmp_txs)
-
-    # build batch cosmos tx
-    cosmos_tx = {
-        "body": {
-            "messages": msgs,
-            "memo": "",
-            "timeout_height": "0",
-            "extension_options": [
-                {"@type": "/ethermint.evm.v1.ExtensionOptionsEthereumTx"}
-            ],
-            "non_critical_extension_options": [],
-        },
-        "auth_info": {
-            "signer_infos": [],
-            "fee": {
-                "amount": [{"denom": "basetcro", "amount": str(fee)}],
-                "gas_limit": str(gas_limit),
-                "payer": "",
-                "granter": "",
-            },
-        },
-        "signatures": [],
-    }
-    with tempfile.NamedTemporaryFile("w") as fp:
-        json.dump(cosmos_tx, fp)
-        fp.flush()
-        rsp = cronos.cosmos_cli().broadcast_tx(fp.name)
-        assert rsp["code"] == 0, rsp["raw_log"]
+    cosmos_tx, tx_hashes = build_batch_tx(
+        w3, cli, [deploy_tx, transfer_tx1, transfer_tx2]
+    )
+    rsp = cli.broadcast_tx_json(cosmos_tx)
+    assert rsp["code"] == 0, rsp["raw_log"]
 
-    receipts = [
-        w3.eth.wait_for_transaction_receipt(signed.hash) for signed in signed_txs
-    ]
+    receipts = [w3.eth.wait_for_transaction_receipt(h) for h in tx_hashes]
 
     assert 2000 == contract.caller.balanceOf(recipient)
 
@@ -548,10 +512,8 @@ def test_batch_tx(cronos):
 
     # check traceTransaction
     rsps = [
-        w3.provider.make_request("debug_traceTransaction", [signed.hash.hex()])[
-            "result"
-        ]
-        for signed in signed_txs
+        w3.provider.make_request("debug_traceTransaction", [h.hex()])["result"]
+        for h in tx_hashes
     ]
 
     for rsp, receipt in zip(rsps, receipts):
@@ -562,8 +524,61 @@ def test_batch_tx(cronos):
     txs = [
         w3.eth.get_transaction_by_block(receipts[0].blockNumber, i) for i in range(3)
     ]
-    for tx, signed in zip(txs, signed_txs):
-        assert tx.hash == signed.hash
+    for tx, h in zip(txs, tx_hashes):
+        assert tx.hash == h
+
+
+def test_failed_transfer_tx(cronos):
+    """
+    It's possible to include a failed transfer transaction in batch tx
+    """
+    w3 = cronos.w3
+    cli = cronos.cosmos_cli()
+    sender = ADDRS["community"]
+    recipient = ADDRS["validator"]
+    nonce = w3.eth.get_transaction_count(sender)
+    half_balance = w3.eth.get_balance(sender) // 3 + 1
+
+    # build batch tx, the third tx will fail, but will be included in block
+    # because of the batch tx.
+    transfer1 = {"from": sender, "nonce": nonce, "to": recipient, "value": half_balance}
+    transfer2 = {
+        "from": sender,
+        "nonce": nonce + 1,
+        "to": recipient,
+        "value": half_balance,
+    }
+    transfer3 = {
+        "from": sender,
+        "nonce": nonce + 2,
+        "to": recipient,
+        "value": half_balance,
+    }
+    cosmos_tx, tx_hashes = build_batch_tx(
+        w3, cli, [transfer1, transfer2, transfer3], KEYS["community"]
+    )
+    rsp = cli.broadcast_tx_json(cosmos_tx)
+    assert rsp["code"] == 0, rsp["raw_log"]
+
+    receipts = [w3.eth.wait_for_transaction_receipt(h) for h in tx_hashes]
+    assert receipts[0].status == receipts[1].status == 1
+    assert receipts[2].status == 0
+
+    # test the cronos_getTransactionReceiptsByBlock api
+    rsp = get_receipts_by_block(w3, receipts[0].blockNumber)
+    assert "error" not in rsp, rsp["error"]
+    assert len(receipts) == len(rsp["result"])
+    for a, b in zip(receipts, rsp["result"]):
+        assert a == b
+
+    # check traceTransaction
+    rsps = [
+        w3.provider.make_request("debug_traceTransaction", [h.hex()])["result"]
+        for h in tx_hashes
+    ]
+    for rsp, receipt in zip(rsps, receipts):
+        assert receipt.status == (not rsp["failed"])
+        assert receipt.gasUsed == rsp["gas"]
 
 
 def test_log0(cluster):

integration_tests/utils.py

@@ -17,7 +17,9 @@
 from eth_account import Account
 from hexbytes import HexBytes
 from pystarport import ledger
+from web3._utils.method_formatters import receipt_formatter
 from web3._utils.transactions import fill_nonce, fill_transaction_defaults
+from web3.datastructures import AttributeDict
 
 load_dotenv(Path(__file__).parent.parent / "scripts/.env")
 Account.enable_unaudited_hdwallet_features()
@@ -393,3 +395,49 @@ def modify_command_in_supervisor_config(ini: Path, fn, **kwargs):
             **kwargs,
         )
     )
+
+
+def build_batch_tx(w3, cli, txs, key=KEYS["validator"]):
+    "return cosmos batch tx and eth tx hashes"
+    signed_txs = [sign_transaction(w3, tx, key) for tx in txs]
+    tmp_txs = [cli.build_evm_tx(signed.rawTransaction.hex()) for signed in signed_txs]
+
+    msgs = [tx["body"]["messages"][0] for tx in tmp_txs]
+    fee = sum(int(tx["auth_info"]["fee"]["amount"][0]["amount"]) for tx in tmp_txs)
+    gas_limit = sum(int(tx["auth_info"]["fee"]["gas_limit"]) for tx in tmp_txs)
+
+    tx_hashes = [signed.hash for signed in signed_txs]
+
+    # build batch cosmos tx
+    return {
+        "body": {
+            "messages": msgs,
+            "memo": "",
+            "timeout_height": "0",
+            "extension_options": [
+                {"@type": "/ethermint.evm.v1.ExtensionOptionsEthereumTx"}
+            ],
+            "non_critical_extension_options": [],
+        },
+        "auth_info": {
+            "signer_infos": [],
+            "fee": {
+                "amount": [{"denom": "basetcro", "amount": str(fee)}],
+                "gas_limit": str(gas_limit),
+                "payer": "",
+                "granter": "",
+            },
+        },
+        "signatures": [],
+    }, tx_hashes
+
+
+def get_receipts_by_block(w3, blk):
+    if isinstance(blk, int):
+        blk = hex(blk)
+    rsp = w3.provider.make_request("cronos_getTransactionReceiptsByBlock", [blk])
+    if "error" not in rsp:
+        rsp["result"] = [
+            AttributeDict(receipt_formatter(item)) for item in rsp["result"]
+        ]
+    return rsp

scripts/cronos-devnet.yaml

@@ -7,7 +7,7 @@ cronos_777-1:
     json-rpc:
       address: "0.0.0.0:{EVMRPC_PORT}"
       ws-address: "0.0.0.0:{EVMRPC_PORT_WS}"
-      api: "eth,net,web3,debug"
+      api: "eth,net,web3,debug,cronos"
   validators:
     - coins: 1000000000000000000stake,10000000000000000000000basetcro
       staked: 1000000000000000000stake

scripts/devnet.yaml

@@ -6,7 +6,7 @@ cronos_777-1:
     json-rpc:
       address: "0.0.0.0:{EVMRPC_PORT}"
       ws-address: "0.0.0.0:{EVMRPC_PORT_WS}"
-      api: "eth,net,web3,debug"
+      api: "eth,net,web3,debug,cronos"
   validators:
     - coins: 1000000000000000000stake,1000000000000000000basetcro
       staked: 1000000000000000000stake