Fix the issue that may occur when using a dictionary to crawl blocks. (…

…#2648) * Fix the issue that may occur when using a dictionary to crawl blocks. * fix unit test * unit test
subquery · Jan 25, 2025 · efdd476 · efdd476
1 parent 8821937
commit efdd476
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 5 deletions.
diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md
@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Fixed the inconsistency between the `monitor-file-size` flag and the expected behavior.(#2644)
 - Improved block range validation in POI endpoint with custom class-validator decorator
+- When setting a smaller batch size and the current processing height reaches latestFinalizedHeight, it causes the program to exit unexpectedly.
 
 ## [16.1.0] - 2024-12-11
 ### Changed

diff --git a/packages/node-core/src/configure/NodeConfig.spec.ts b/packages/node-core/src/configure/NodeConfig.spec.ts
@@ -48,7 +48,7 @@ describe('NodeConfig', () => {
   it('Monitor configs default value', () => {
     const fileConfig = NodeConfig.fromFile(path.join(__dirname, '../../test/config.yml'));
     expect(fileConfig.monitorFileSize).toEqual(0);
-    expect(fileConfig.monitorObjectMaxDepth).toEqual(0);
+    expect(fileConfig.monitorObjectMaxDepth).toEqual(5);
 
     const config2 = NodeConfig.rebaseWithArgs(fileConfig, {
       monitorObjectMaxDepth: 10,

diff --git a/packages/node-core/src/indexer/dictionary/v1/dictionaryV1.spec.ts b/packages/node-core/src/indexer/dictionary/v1/dictionaryV1.spec.ts
@@ -10,7 +10,7 @@ import {BlockHeightMap} from '../../../utils/blockHeightMap';
 import {dsMap, mockDS, TestDictionaryV1, HAPPY_PATH_CONDITIONS} from '../dictionary.fixtures';
 import {getGqlType} from './utils';
 
-const DICTIONARY_ENDPOINT = `https://gateway.subquery.network/query/QmUGBdhQKnzE8q6x6MPqP6LNZGa8gzXf5gkdmhzWjdFGfL`;
+const DICTIONARY_ENDPOINT = `https://gateway.subquery.network/query/QmSxAgGGpaMrYzooWpydmwzutREwomL5nupLZqxURzuJTo`;
 const DICTIONARY_CHAINID = `0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3`;
 
 const nodeConfig = new NodeConfig({

diff --git a/packages/node-core/src/indexer/fetch.service.spec.ts b/packages/node-core/src/indexer/fetch.service.spec.ts
@@ -719,4 +719,34 @@ describe('Fetch Service', () => {
     // when last processed height is 1000, finalized height is 1000
     await expect(fetchService.init(1001)).resolves.not.toThrow();
   });
+
+  it('When the index height reaches the dictionary’s lastBufferedHeight, it can enqueue normally.', async () => {
+    enableDictionary();
+    dictionaryService.scopedDictionaryEntries = (start, end, batch) => {
+      return Promise.resolve({
+        batchBlocks: [],
+        queryEndBlock: 900,
+        _metadata: {
+          lastProcessedHeight: 900,
+        } as any,
+        lastBufferedHeight: 900,
+      });
+    };
+
+    fetchService.bestHeight = 1000;
+    const dictionarySpy = jest.spyOn((fetchService as any).dictionaryService, 'scopedDictionaryEntries');
+
+    // first enqueue
+    await fetchService.init(10);
+
+    expect(dictionarySpy).toHaveBeenCalledTimes(1);
+    expect((fetchService as any).blockDispatcher.latestBufferedHeight).toEqual(900);
+
+    // Second enqueue
+    blockDispatcher.freeSize = 10;
+    await delay(1);
+    expect(dictionarySpy).toHaveBeenCalledTimes(2);
+    expect(spyOnEnqueueSequential).toHaveBeenCalledTimes(1);
+    expect((fetchService as any).blockDispatcher.latestBufferedHeight).toEqual(910);
+  }, 10000);
 });
diff --git a/packages/node-core/src/indexer/fetch.service.ts b/packages/node-core/src/indexer/fetch.service.ts
@@ -241,15 +241,20 @@ export abstract class BaseFetchService<DS extends BaseDataSource, B extends IBlo
             continue;
           }
           if (dictionary) {
-            const {batchBlocks} = dictionary;
+            const {batchBlocks, lastBufferedHeight} = dictionary;
             // the last block returned from batch should have max height in this batch
             const mergedBlocks = mergeNumAndBlocks(
-              this.getModuloBlocks(startBlockHeight, dictionary.lastBufferedHeight),
+              this.getModuloBlocks(startBlockHeight, lastBufferedHeight),
               batchBlocks
             );
             if (mergedBlocks.length === 0) {
               // There we're no blocks in this query range, we can set a new height we're up to
-              await this.enqueueBlocks([], dictionary.lastBufferedHeight);
+              if (startBlockHeight <= lastBufferedHeight) {
+                await this.enqueueBlocks([], lastBufferedHeight);
+              } else {
+                // Exceeds the dictionary search height
+                await this.enqueueSequential(startBlockHeight, scaledBatchSize, latestHeight);
+              }
             } else {
               const maxBlockSize = Math.min(mergedBlocks.length, this.blockDispatcher.freeSize);
               const enqueueBlocks = mergedBlocks.slice(0, maxBlockSize);