Handle rate limit

packages/node/src/avalanche/api.avalanche.ts

@@ -8,18 +8,20 @@ import https from 'https';
 import { Interface } from '@ethersproject/abi';
 import { hexDataSlice } from '@ethersproject/bytes';
 import { RuntimeDataSourceV0_2_0 } from '@subql/common-avalanche';
-import { getLogger } from '@subql/node-core';
+import { delay, getLogger } from '@subql/node-core';
 import {
   ApiWrapper,
   AvalancheLog,
   AvalancheBlockWrapper,
   AvalancheTransaction,
   AvalancheResult,
   BlockWrapper,
+  AvalancheBlock,
 } from '@subql/types-avalanche';
 import { Avalanche } from 'avalanche';
 import { EVMAPI } from 'avalanche/dist/apis/evm';
 import { IndexAPI } from 'avalanche/dist/apis/index';
+import { RequestResponseData } from 'avalanche/typings/src/common';
 import { BigNumber } from 'ethers';
 import { AvalancheBlockWrapped } from './block.avalanche';
 import { CChainProvider } from './provider';
@@ -60,6 +62,8 @@ async function loadAssets(
   return res;
 }
 
+const RETRY_COUNT = 5;
+
 export class AvalancheApi implements ApiWrapper<AvalancheBlockWrapper> {
   private client: Avalanche;
   private indexApi: IndexAPI;
@@ -69,6 +73,7 @@ export class AvalancheApi implements ApiWrapper<AvalancheBlockWrapper> {
   private cchain: EVMAPI;
   private contractInterfaces: Record<string, Interface> = {};
   private chainId: string;
+  // private retryCount = 25;
 
   constructor(private options: AvalancheOptions) {
     this.encoding = 'cb58';
@@ -170,35 +175,89 @@ export class AvalancheApi implements ApiWrapper<AvalancheBlockWrapper> {
     return BigNumber.from(res.data.result).toNumber();
   }
 
+  async callMethod(
+    method: string,
+    params: object[] | object,
+    retries: number,
+  ): Promise<RequestResponseData> {
+    try {
+      return await this.cchain.callMethod(
+        method,
+        params,
+        `/ext/bc/${this.options.subnet}/rpc`,
+      );
+    } catch (e) {
+      if (e.response.status !== 429) {
+        throw e;
+      }
+
+      // If callMethod failed due to 429, retry the request
+      if (retries > 0) {
+        logger.warn(`Retrying request (${retries}), due to 429 status code`);
+        --retries;
+        await delay(10);
+        return this.callMethod(method, params, retries);
+      } else {
+        const error = new Error(e.message);
+        logger.error(error, `Retry: ${retries} failed`);
+        throw e;
+      }
+    }
+  }
+
+  async transactionReceipts(
+    tx: AvalancheTransaction,
+    num: number,
+    block: AvalancheBlock,
+  ): Promise<AvalancheTransaction<AvalancheResult>> {
+    try {
+      const transaction = formatTransaction(tx);
+
+      const receipt = (
+        await this.callMethod(
+          'eth_getTransactionReceipt',
+          [tx.hash],
+          RETRY_COUNT,
+        )
+      ).data.result;
+      transaction.receipt = formatReceipt(receipt, block);
+      return transaction;
+    } catch (e) {
+      const error = new Error(e.message);
+      logger.error(error, `Failed to fetch blockTransaction at ${num}`);
+      throw e;
+    }
+  }
+
+  async fetchBlock(num: number): Promise<any> {
+    try {
+      const block_promise = await this.callMethod(
+        'eth_getBlockByNumber',
+        [`0x${num.toString(16)}`, true],
+        RETRY_COUNT,
+      );
+
+      const block = formatBlock(block_promise.data.result);
+
+      // Get transaction receipts
+      block.transactions = await Promise.all(
+        block.transactions.map(async (tx) =>
+          this.transactionReceipts(tx, num, block),
+        ),
+      );
+      return new AvalancheBlockWrapped(block);
+    } catch (e) {
+      const error = new Error(e.message);
+      logger.error(error, `Failed to fetch block at height ${num}`);
+      throw e;
+    }
+  }
+
   async fetchBlocks(bufferBlocks: number[]): Promise<AvalancheBlockWrapper[]> {
     return Promise.all(
       bufferBlocks.map(async (num) => {
         try {
-          // Fetch Block
-          const block_promise = await this.cchain.callMethod(
-            'eth_getBlockByNumber',
-            [`0x${num.toString(16)}`, true],
-            `/ext/bc/${this.options.subnet}/rpc`,
-          );
-
-          const block = formatBlock(block_promise.data.result);
-
-          // Get transaction receipts
-          block.transactions = await Promise.all(
-            block.transactions.map(async (tx) => {
-              const transaction = formatTransaction(tx);
-              const receipt = (
-                await this.cchain.callMethod(
-                  'eth_getTransactionReceipt',
-                  [tx.hash],
-                  `/ext/bc/${this.options.subnet}/rpc`,
-                )
-              ).data.result;
-              transaction.receipt = formatReceipt(receipt, block);
-              return transaction;
-            }),
-          );
-          return new AvalancheBlockWrapped(block);
+          return this.fetchBlock(num);
         } catch (e) {
           // Wrap error from an axios error to fix issue with error being undefined
           const error = new Error(e.message);

packages/node/src/indexer/fetch.service.ts

@@ -280,7 +280,7 @@ export class FetchService implements OnApplicationShutdown {
         });
       }
     } catch (e) {
-      logger.error(e, `Having a problem when get finalized block`);
+      logger.warn(e, `Having a problem when get finalized block`);
     }
   }
 
@@ -299,7 +299,7 @@ export class FetchService implements OnApplicationShutdown {
         });
       }
     } catch (e) {
-      logger.error(e, `Having a problem when get best block`);
+      logger.warn(e, `Having a problem when get best block`);
     }
   }