feat: add refuel in proto and updatae kafka consumer configs

0xPolygon · Oct 13, 2023 · 0148214 · 0148214
1 parent 1ec7530
commit 0148214
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 16 deletions.
diff --git a/examples/README.md b/examples/README.md
@@ -14,24 +14,48 @@ The folder consists of 2 examples and each project consists of three distinct pa
 Both the examples serves as a useful reference for understanding how Chain Indexer Framework can be effectively utilized to manage and process blockchain data efficiently.
 
 
-## How to Build
+## How to Build and Run
 
-This project can build on Windows, MacOs and Linux
+This project can be built and run on Windows, MacOS, and Linux. To get started, follow these steps:
 
-Steps to build specifically on Ubuntu are as follows:
+**Step 1: Install Kafka**
 
-- Install docker and check version once installed
-  - ```bash
-    docker --version
-    ```
+You have two options for running Kafka, either inside a Docker container or locally without Docker.
+
+Option 1: **Inside Docker** 
+  - Install Docker and verify the installation:
+    - ```bash
+      docker --version
+      ```
 
-- Run docker compose from dev-env. Make sure that docker daemon is running.
-  - ```bash
-    cd dev-env
-    docker compose up -d
-    ```
+  - Run Docker Compose from the "dev-env" directory. Make sure that the Docker daemon is running:
+    - ```bash
+      cd dev-env
+      docker compose up -d
+      ```
+
+Option 2: **Run Locally without Docker**
+  - Download kafka from [here](https://www.apache.org/dyn/closer.cgi?path=/kafka/3.6.0/kafka_2.13-3.6.0.tgz)
+
+  - Move to the Kafka folder you downloaded:
+    - ```bash
+      cd kafka_2.13-3.6.0
+      ```
+
+  - Run Zookeeper:
+    - ```bash
+      bin/zookeeper-server-start.sh config/zookeeper.properties
+      ```
+
+  - Run Kafka:
+    - ```bash
+      bin/kafka-server-start.sh config/server.properties
+      ```
+
+**Step 2: Install and Run MongoDB**
+
+Ensure that MongoDB is installed and its service is running. Follow these steps to install MongoDB:
 
-- Make sure MongoDb is installed and the service is running
   - ```bash
     curl -fsSL https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add -
     apt-key list
@@ -41,7 +65,9 @@ Steps to build specifically on Ubuntu are as follows:
     sudo systemctl start mongod.service
     sudo systemctl status mongod
     ```
-
+
+**Step 3: Build and Run Services**
+
 - Run [MATIC_TRANSFER](./matic_transfer/README.md)
 
 - Run [NFT_BALANCER](./nft_balancer/README.md)

diff --git a/internal/interfaces/deposit.ts b/internal/interfaces/deposit.ts
@@ -12,4 +12,5 @@ export interface IDeposit {
     tokenIds?: string[],
     timestamp?: number,
     rootTunnelAddress?: string
+    refuel?: boolean
 }
diff --git a/internal/kafka/consumer/abstract_consumer.ts b/internal/kafka/consumer/abstract_consumer.ts
@@ -73,7 +73,8 @@ export abstract class AbstractConsumer extends noderdkafka.KafkaConsumer {
                 "event_cb": true,
                 "message.max.bytes": 26214400,
                 "fetch.message.max.bytes": 26214400,
-                "queued.max.messages.kbytes": 25000
+                "queued.max.messages.kbytes": 25000,
+                "isolation.level": "read_uncommitted",
             }, config),
             Object.assign(
                 {

diff --git a/schemas/bridge_assets.proto b/schemas/bridge_assets.proto
@@ -36,6 +36,7 @@ message BridgeAssets {
         ExitRoots exitRoots = 8;
         string depositor = 9;
         uint32 leafType = 10;
+        bool refuel = 11;
     }
 
     uint64 blockNumber = 1;

diff --git a/schemas/depositblock.proto b/schemas/depositblock.proto
@@ -12,6 +12,7 @@ message DepositBlock {
         repeated string amounts = 7;
         repeated uint64 tokenIds = 8;
         string rootTunnelAddress = 9;
+        bool refuel = 10;
     }
 
     required uint64 blockNumber = 1;

diff --git a/tests/kafka/consumer/abstract_consumer.test.ts b/tests/kafka/consumer/abstract_consumer.test.ts
@@ -85,6 +85,7 @@ describe("Kafka - AbstractConsumer", () => {
                     "enable.auto.offset.store": false,
                     "event_cb": false,
                     "message.max.bytes": 26214400,
+                    "isolation.level": "read_uncommitted",
                     "fetch.message.max.bytes": 26214400,
                     "queued.max.messages.kbytes": 25000
                 },
@@ -105,7 +106,8 @@ describe("Kafka - AbstractConsumer", () => {
                     "event_cb": true,
                     "message.max.bytes": 26214400,
                     "fetch.message.max.bytes": 26214400,
-                    "queued.max.messages.kbytes": 25000
+                    "queued.max.messages.kbytes": 25000,
+                    "isolation.level": "read_uncommitted",
                 },
                 {
                     "auto.offset.reset": "earliest" as "earliest" | "smallest" | "beginning" | "largest" | "latest" | "end" | "error" | undefined