From 5a7912132da4af42b992fec386112ae0749bd72c Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Thu, 2 Jan 2025 10:41:44 +0500 Subject: [PATCH 01/10] build: multi block ranges point to multi sources --- .../src/build/configAndIndexingFunctions.ts | 541 ++++++++++-------- packages/core/src/config/config.ts | 12 +- packages/core/src/types/virtual.ts | 8 +- 3 files changed, 303 insertions(+), 258 deletions(-) diff --git a/packages/core/src/build/configAndIndexingFunctions.ts b/packages/core/src/build/configAndIndexingFunctions.ts index 7d77c72a8..c2a542124 100644 --- a/packages/core/src/build/configAndIndexingFunctions.ts +++ b/packages/core/src/build/configAndIndexingFunctions.ts @@ -63,6 +63,60 @@ const flattenSources = < ); }; +function resolveBlockRanges( + blocks: [number | undefined, number | undefined][] | undefined, +): [number | undefined, number | undefined][] { + const blockRanges: [number | undefined, number | undefined][] = + blocks === undefined || blocks.length === 0 + ? [[undefined, undefined]] + : blocks; + blockRanges.sort((a, b) => { + if (a[0] === undefined) { + return -1; + } + if (b[0] === undefined) { + return 1; + } + return a[0] - b[0]; + }); + + const resolvedBlockRanges: [number | undefined, number | undefined][] = []; + + for (const blockRange of blockRanges) { + const [curStartBlockMaybeNaN, curEndBlockMaybeNaN] = blockRange; + const curStartBlock = Number.isNaN(curStartBlockMaybeNaN) + ? undefined + : curStartBlockMaybeNaN; + const curEndBlock = Number.isNaN(curEndBlockMaybeNaN) + ? undefined + : curEndBlockMaybeNaN; + + if (resolvedBlockRanges.length === 0) { + resolvedBlockRanges.push([curStartBlock, curEndBlock]); + continue; + } + + const last = resolvedBlockRanges[resolvedBlockRanges.length - 1]!; + const [lastStartBlock, lastEndBlock] = last; + if (lastEndBlock === undefined) { + break; + } + // Check for overlap + if (curStartBlock === undefined || curStartBlock <= lastEndBlock) { + resolvedBlockRanges[resolvedBlockRanges.length - 1] = [ + lastStartBlock, + curEndBlock === undefined || curEndBlock >= lastEndBlock + ? curEndBlock + : lastEndBlock, + ]; + } else { + resolvedBlockRanges.push([curStartBlock, curEndBlock]); + } + } + + return resolvedBlockRanges; +} + export async function buildConfigAndIndexingFunctions({ config, rawIndexingFunctions, @@ -221,23 +275,26 @@ export async function buildConfigAndIndexingFunctions({ ); } - const startBlockMaybeNan = source.startBlock; - const startBlock = Number.isNaN(startBlockMaybeNan) - ? undefined - : startBlockMaybeNan; - const endBlockMaybeNan = source.endBlock; - const endBlock = Number.isNaN(endBlockMaybeNan) - ? undefined - : endBlockMaybeNan; - - if ( - startBlock !== undefined && - endBlock !== undefined && - endBlock < startBlock - ) { - throw new Error( - `Validation failed: Start block for '${source.name}' is after end block (${startBlock} > ${endBlock}).`, - ); + const blockRanges = + source.blocks === undefined ? [[undefined, undefined]] : source.blocks; + for (const blockRange of blockRanges) { + const startBlockMaybeNan = blockRange[0]; + const startBlock = Number.isNaN(startBlockMaybeNan) + ? undefined + : startBlockMaybeNan; + const endBlockMaybeNan = blockRange[1]; + const endBlock = Number.isNaN(endBlockMaybeNan) + ? undefined + : endBlockMaybeNan; + if ( + startBlock !== undefined && + endBlock !== undefined && + endBlock < startBlock + ) { + throw new Error( + `Validation failed: Start block for '${source.name}' is after end block (${startBlock} > ${endBlock}).`, + ); + } } const network = networks.find((n) => n.name === source.network); @@ -381,14 +438,7 @@ export async function buildConfigAndIndexingFunctions({ }); } - const startBlockMaybeNan = source.startBlock; - const fromBlock = Number.isNaN(startBlockMaybeNan) - ? undefined - : startBlockMaybeNan; - const endBlockMaybeNan = source.endBlock; - const toBlock = Number.isNaN(endBlockMaybeNan) - ? undefined - : endBlockMaybeNan; + const resolvedBlockRanges = resolveBlockRanges(source.blocks); const contractMetadata = { type: "contract", @@ -411,14 +461,92 @@ export async function buildConfigAndIndexingFunctions({ ...resolvedAddress, }); - const logSources = topicsArray.map( - (topics) => + const logSources = topicsArray.flatMap((topics) => + resolvedBlockRanges.map( + ([fromBlock, toBlock]) => + ({ + ...contractMetadata, + filter: { + type: "log", + chainId: network.chainId, + address: logFactory, + topic0: topics.topic0, + topic1: topics.topic1, + topic2: topics.topic2, + topic3: topics.topic3, + fromBlock, + toBlock, + include: defaultLogFilterInclude.concat( + source.includeTransactionReceipts + ? defaultTransactionReceiptInclude + : [], + ), + }, + }) satisfies ContractSource, + ), + ); + + if (source.includeCallTraces) { + const callTraceSources = resolvedBlockRanges.map( + ([fromBlock, toBlock]) => + ({ + ...contractMetadata, + filter: { + type: "trace", + chainId: network.chainId, + fromAddress: undefined, + toAddress: logFactory, + callType: "CALL", + functionSelector: registeredFunctionSelectors, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTraceFilterInclude.concat( + source.includeTransactionReceipts + ? defaultTransactionReceiptInclude + : [], + ), + }, + }) satisfies ContractSource, + ); + + return [...logSources, ...callTraceSources]; + } + + return logSources; + } else if (resolvedAddress !== undefined) { + for (const address of Array.isArray(resolvedAddress) + ? resolvedAddress + : [resolvedAddress]) { + if (!address!.startsWith("0x")) + throw new Error( + `Validation failed: Invalid prefix for address '${address}'. Got '${address!.slice( + 0, + 2, + )}', expected '0x'.`, + ); + if (address!.length !== 42) + throw new Error( + `Validation failed: Invalid length for address '${address}'. Got ${address!.length}, expected 42 characters.`, + ); + } + } + + const validatedAddress = Array.isArray(resolvedAddress) + ? (resolvedAddress.map((r) => toLowerCase(r)) as Address[]) + : resolvedAddress !== undefined + ? (toLowerCase(resolvedAddress) as Address) + : undefined; + + const logSources = topicsArray.flatMap((topics) => + resolvedBlockRanges.map( + ([fromBlock, toBlock]) => ({ ...contractMetadata, filter: { type: "log", chainId: network.chainId, - address: logFactory, + address: validatedAddress, topic0: topics.topic0, topic1: topics.topic1, topic2: topics.topic2, @@ -432,18 +560,23 @@ export async function buildConfigAndIndexingFunctions({ ), }, }) satisfies ContractSource, - ); + ), + ); - if (source.includeCallTraces) { - return [ - ...logSources, - { + if (source.includeCallTraces) { + const callTraceSources = resolvedBlockRanges.map( + ([fromBlock, toBlock]) => + ({ ...contractMetadata, filter: { type: "trace", chainId: network.chainId, fromAddress: undefined, - toAddress: logFactory, + toAddress: Array.isArray(validatedAddress) + ? validatedAddress + : validatedAddress === undefined + ? undefined + : [validatedAddress], callType: "CALL", functionSelector: registeredFunctionSelectors, includeReverted: false, @@ -455,85 +588,10 @@ export async function buildConfigAndIndexingFunctions({ : [], ), }, - } satisfies ContractSource, - ]; - } - - return logSources; - } else if (resolvedAddress !== undefined) { - for (const address of Array.isArray(resolvedAddress) - ? resolvedAddress - : [resolvedAddress]) { - if (!address!.startsWith("0x")) - throw new Error( - `Validation failed: Invalid prefix for address '${address}'. Got '${address!.slice( - 0, - 2, - )}', expected '0x'.`, - ); - if (address!.length !== 42) - throw new Error( - `Validation failed: Invalid length for address '${address}'. Got ${address!.length}, expected 42 characters.`, - ); - } - } - - const validatedAddress = Array.isArray(resolvedAddress) - ? (resolvedAddress.map((r) => toLowerCase(r)) as Address[]) - : resolvedAddress !== undefined - ? (toLowerCase(resolvedAddress) as Address) - : undefined; - - const logSources = topicsArray.map( - (topics) => - ({ - ...contractMetadata, - filter: { - type: "log", - chainId: network.chainId, - address: validatedAddress, - topic0: topics.topic0, - topic1: topics.topic1, - topic2: topics.topic2, - topic3: topics.topic3, - fromBlock, - toBlock, - include: defaultLogFilterInclude.concat( - source.includeTransactionReceipts - ? defaultTransactionReceiptInclude - : [], - ), - }, - }) satisfies ContractSource, - ); + }) satisfies ContractSource, + ); - if (source.includeCallTraces) { - return [ - ...logSources, - { - ...contractMetadata, - filter: { - type: "trace", - chainId: network.chainId, - fromAddress: undefined, - toAddress: Array.isArray(validatedAddress) - ? validatedAddress - : validatedAddress === undefined - ? undefined - : [validatedAddress], - callType: "CALL", - functionSelector: registeredFunctionSelectors, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTraceFilterInclude.concat( - source.includeTransactionReceipts - ? defaultTransactionReceiptInclude - : [], - ), - }, - } satisfies ContractSource, - ]; + return [...logSources, ...callTraceSources]; } else return logSources; }) // Remove sources with no registered indexing functions .filter((source) => { @@ -558,14 +616,7 @@ export async function buildConfigAndIndexingFunctions({ .flatMap((source): AccountSource[] => { const network = networks.find((n) => n.name === source.network)!; - const startBlockMaybeNan = source.startBlock; - const fromBlock = Number.isNaN(startBlockMaybeNan) - ? undefined - : startBlockMaybeNan; - const endBlockMaybeNan = source.endBlock; - const toBlock = Number.isNaN(endBlockMaybeNan) - ? undefined - : endBlockMaybeNan; + const resolvedBlockRanges = resolveBlockRanges(source.blocks); const resolvedAddress = source?.address; @@ -585,16 +636,116 @@ export async function buildConfigAndIndexingFunctions({ ...resolvedAddress, }); - return [ + const accountSources = resolvedBlockRanges.flatMap( + ([fromBlock, toBlock]) => [ + { + type: "account", + name: source.name, + networkName: source.network, + filter: { + type: "transaction", + chainId: network.chainId, + fromAddress: undefined, + toAddress: logFactory, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTransactionFilterInclude, + }, + } satisfies AccountSource, + { + type: "account", + name: source.name, + networkName: source.network, + filter: { + type: "transaction", + chainId: network.chainId, + fromAddress: logFactory, + toAddress: undefined, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTransactionFilterInclude, + }, + } satisfies AccountSource, + { + type: "account", + name: source.name, + networkName: source.network, + filter: { + type: "transfer", + chainId: network.chainId, + fromAddress: undefined, + toAddress: logFactory, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTransferFilterInclude.concat( + source.includeTransactionReceipts + ? defaultTransactionReceiptInclude + : [], + ), + }, + } satisfies AccountSource, + { + type: "account", + name: source.name, + networkName: source.network, + filter: { + type: "transfer", + chainId: network.chainId, + fromAddress: logFactory, + toAddress: undefined, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTransferFilterInclude.concat( + source.includeTransactionReceipts + ? defaultTransactionReceiptInclude + : [], + ), + }, + } satisfies AccountSource, + ], + ); + + return accountSources; + } + + for (const address of Array.isArray(resolvedAddress) + ? resolvedAddress + : [resolvedAddress]) { + if (!address!.startsWith("0x")) + throw new Error( + `Validation failed: Invalid prefix for address '${address}'. Got '${address!.slice( + 0, + 2, + )}', expected '0x'.`, + ); + if (address!.length !== 42) + throw new Error( + `Validation failed: Invalid length for address '${address}'. Got ${address!.length}, expected 42 characters.`, + ); + } + + const validatedAddress = Array.isArray(resolvedAddress) + ? (resolvedAddress.map((r) => toLowerCase(r)) as Address[]) + : resolvedAddress !== undefined + ? (toLowerCase(resolvedAddress) as Address) + : undefined; + + const accountSources = resolvedBlockRanges.flatMap( + ([fromBlock, toBlock]) => [ { type: "account", name: source.name, + networkName: source.network, filter: { type: "transaction", chainId: network.chainId, fromAddress: undefined, - toAddress: logFactory, + toAddress: validatedAddress, includeReverted: false, fromBlock, toBlock, @@ -608,7 +759,7 @@ export async function buildConfigAndIndexingFunctions({ filter: { type: "transaction", chainId: network.chainId, - fromAddress: logFactory, + fromAddress: validatedAddress, toAddress: undefined, includeReverted: false, fromBlock, @@ -624,7 +775,7 @@ export async function buildConfigAndIndexingFunctions({ type: "transfer", chainId: network.chainId, fromAddress: undefined, - toAddress: logFactory, + toAddress: validatedAddress, includeReverted: false, fromBlock, toBlock, @@ -642,7 +793,7 @@ export async function buildConfigAndIndexingFunctions({ filter: { type: "transfer", chainId: network.chainId, - fromAddress: logFactory, + fromAddress: validatedAddress, toAddress: undefined, includeReverted: false, fromBlock, @@ -654,102 +805,10 @@ export async function buildConfigAndIndexingFunctions({ ), }, } satisfies AccountSource, - ]; - } - - for (const address of Array.isArray(resolvedAddress) - ? resolvedAddress - : [resolvedAddress]) { - if (!address!.startsWith("0x")) - throw new Error( - `Validation failed: Invalid prefix for address '${address}'. Got '${address!.slice( - 0, - 2, - )}', expected '0x'.`, - ); - if (address!.length !== 42) - throw new Error( - `Validation failed: Invalid length for address '${address}'. Got ${address!.length}, expected 42 characters.`, - ); - } - - const validatedAddress = Array.isArray(resolvedAddress) - ? (resolvedAddress.map((r) => toLowerCase(r)) as Address[]) - : resolvedAddress !== undefined - ? (toLowerCase(resolvedAddress) as Address) - : undefined; + ], + ); - return [ - { - type: "account", - name: source.name, - - networkName: source.network, - filter: { - type: "transaction", - chainId: network.chainId, - fromAddress: undefined, - toAddress: validatedAddress, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTransactionFilterInclude, - }, - } satisfies AccountSource, - { - type: "account", - name: source.name, - networkName: source.network, - filter: { - type: "transaction", - chainId: network.chainId, - fromAddress: validatedAddress, - toAddress: undefined, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTransactionFilterInclude, - }, - } satisfies AccountSource, - { - type: "account", - name: source.name, - networkName: source.network, - filter: { - type: "transfer", - chainId: network.chainId, - fromAddress: undefined, - toAddress: validatedAddress, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTransferFilterInclude.concat( - source.includeTransactionReceipts - ? defaultTransactionReceiptInclude - : [], - ), - }, - } satisfies AccountSource, - { - type: "account", - name: source.name, - networkName: source.network, - filter: { - type: "transfer", - chainId: network.chainId, - fromAddress: validatedAddress, - toAddress: undefined, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTransferFilterInclude.concat( - source.includeTransactionReceipts - ? defaultTransactionReceiptInclude - : [], - ), - }, - } satisfies AccountSource, - ]; + return accountSources; }) .filter((source) => { const eventName = @@ -773,7 +832,7 @@ export async function buildConfigAndIndexingFunctions({ }); const blockSources: BlockSource[] = flattenSources(config.blocks ?? {}) - .map((source) => { + .flatMap((source) => { const network = networks.find((n) => n.name === source.network)!; const intervalMaybeNan = source.interval ?? 1; @@ -785,29 +844,25 @@ export async function buildConfigAndIndexingFunctions({ ); } - const startBlockMaybeNan = source.startBlock; - const fromBlock = Number.isNaN(startBlockMaybeNan) - ? undefined - : startBlockMaybeNan; - const endBlockMaybeNan = source.endBlock; - const toBlock = Number.isNaN(endBlockMaybeNan) - ? undefined - : endBlockMaybeNan; + const resolvedBlockRanges = resolveBlockRanges(source.blocks); - return { - type: "block", - name: source.name, - networkName: source.network, - filter: { - type: "block", - chainId: network.chainId, - interval: interval, - offset: (fromBlock ?? 0) % interval, - fromBlock, - toBlock, - include: defaultBlockFilterInclude, - }, - } satisfies BlockSource; + return resolvedBlockRanges.map( + ([fromBlock, toBlock]) => + ({ + type: "block", + name: source.name, + networkName: source.network, + filter: { + type: "block", + chainId: network.chainId, + interval: interval, + offset: (fromBlock ?? 0) % interval, + fromBlock, + toBlock, + include: defaultBlockFilterInclude, + }, + }) satisfies BlockSource, + ); }) .filter((blockSource) => { const hasRegisteredIndexingFunction = diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 082384549..0c68b8387 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -62,10 +62,8 @@ type DatabaseConfig = // base type BlockConfig = { - /** Block number at which to start indexing events (inclusive). If `undefined`, events will be processed from block 0. Default: `undefined`. */ - startBlock?: number; - /** Block number at which to stop indexing events (inclusive). If `undefined`, events will be processed in real-time. Default: `undefined`. */ - endBlock?: number; + /** Block intervals with startBlock (inclusive) and endBlock (inclusive). If `undefined`, events will be processed from block 0 and in real-time. */ + blocks?: [number | undefined, number | undefined][]; }; type TransactionReceiptConfig = { @@ -212,11 +210,7 @@ type AccountsConfig = {} extends accounts // blocks -type BlockFilterConfig = { - /** Block number at which to start indexing events (inclusive). If `undefined`, events will be processed from block 0. Default: `undefined`. */ - startBlock?: number; - /** Block number at which to stop indexing events (inclusive). If `undefined`, events will be processed in real-time. Default: `undefined`. */ - endBlock?: number; +type BlockFilterConfig = BlockConfig & { interval?: number; }; diff --git a/packages/core/src/types/virtual.ts b/packages/core/src/types/virtual.ts index 5d5081016..dcebff030 100644 --- a/packages/core/src/types/virtual.ts +++ b/packages/core/src/types/virtual.ts @@ -200,13 +200,9 @@ export namespace Virtual { config["contracts"][_contractName], "address" >; - startBlock: ExtractOverridenProperty< + blocks: ExtractOverridenProperty< config["contracts"][_contractName], - "startBlock" - >; - endBlock: ExtractOverridenProperty< - config["contracts"][_contractName], - "endBlock" + "blocks" >; }; }; From 535b5b17cc25b3b7e2dce326b7df6f7970fc2064 Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Thu, 2 Jan 2025 17:04:51 +0500 Subject: [PATCH 02/10] realtime toBlock --- .../src/build/configAndIndexingFunctions.ts | 98 ++++++++----------- packages/core/src/config/config.ts | 4 +- packages/core/src/sync/index.ts | 11 +-- packages/core/src/sync/source.ts | 10 +- 4 files changed, 53 insertions(+), 70 deletions(-) diff --git a/packages/core/src/build/configAndIndexingFunctions.ts b/packages/core/src/build/configAndIndexingFunctions.ts index c2a542124..83b569d5a 100644 --- a/packages/core/src/build/configAndIndexingFunctions.ts +++ b/packages/core/src/build/configAndIndexingFunctions.ts @@ -1,5 +1,5 @@ import { BuildError } from "@/common/errors.js"; -import type { Config } from "@/config/config.js"; +import type { BlockRange, Config } from "@/config/config.js"; import { type Network, getFinalityBlockCount, @@ -63,34 +63,20 @@ const flattenSources = < ); }; -function resolveBlockRanges( - blocks: [number | undefined, number | undefined][] | undefined, -): [number | undefined, number | undefined][] { - const blockRanges: [number | undefined, number | undefined][] = +function resolveBlockRanges(blocks: BlockRange[] | undefined): BlockRange[] { + const blockRanges: BlockRange[] = blocks === undefined || blocks.length === 0 - ? [[undefined, undefined]] - : blocks; - blockRanges.sort((a, b) => { - if (a[0] === undefined) { - return -1; - } - if (b[0] === undefined) { - return 1; - } - return a[0] - b[0]; - }); + ? [[0, "realtime"]] + : blocks.map(([rawStartBlock, rawEndBlock]) => [ + Number.isNaN(rawStartBlock) ? 0 : rawStartBlock, + Number.isNaN(rawEndBlock) ? "realtime" : rawEndBlock, + ]); - const resolvedBlockRanges: [number | undefined, number | undefined][] = []; + blockRanges.sort((a, b) => a[0] - b[0]); - for (const blockRange of blockRanges) { - const [curStartBlockMaybeNaN, curEndBlockMaybeNaN] = blockRange; - const curStartBlock = Number.isNaN(curStartBlockMaybeNaN) - ? undefined - : curStartBlockMaybeNaN; - const curEndBlock = Number.isNaN(curEndBlockMaybeNaN) - ? undefined - : curEndBlockMaybeNaN; + const resolvedBlockRanges: BlockRange[] = []; + for (const [curStartBlock, curEndBlock] of blockRanges) { if (resolvedBlockRanges.length === 0) { resolvedBlockRanges.push([curStartBlock, curEndBlock]); continue; @@ -98,14 +84,14 @@ function resolveBlockRanges( const last = resolvedBlockRanges[resolvedBlockRanges.length - 1]!; const [lastStartBlock, lastEndBlock] = last; - if (lastEndBlock === undefined) { + if (lastEndBlock === "realtime") { break; } - // Check for overlap + // Check for overlapping block ranges if (curStartBlock === undefined || curStartBlock <= lastEndBlock) { resolvedBlockRanges[resolvedBlockRanges.length - 1] = [ lastStartBlock, - curEndBlock === undefined || curEndBlock >= lastEndBlock + curEndBlock === "realtime" || curEndBlock >= lastEndBlock ? curEndBlock : lastEndBlock, ]; @@ -275,26 +261,22 @@ export async function buildConfigAndIndexingFunctions({ ); } - const blockRanges = - source.blocks === undefined ? [[undefined, undefined]] : source.blocks; - for (const blockRange of blockRanges) { - const startBlockMaybeNan = blockRange[0]; - const startBlock = Number.isNaN(startBlockMaybeNan) - ? undefined - : startBlockMaybeNan; - const endBlockMaybeNan = blockRange[1]; - const endBlock = Number.isNaN(endBlockMaybeNan) - ? undefined - : endBlockMaybeNan; - if ( - startBlock !== undefined && - endBlock !== undefined && - endBlock < startBlock - ) { + const blockRanges: BlockRange[] = + source.blocks === undefined ? [[0, "realtime"]] : source.blocks; + for (const [rawStartBlock, rawEndBlock] of blockRanges) { + const startBlock = Number.isNaN(rawStartBlock) ? 0 : rawStartBlock; + const endBlock = Number.isNaN(rawEndBlock) ? "realtime" : rawEndBlock; + if (typeof endBlock !== "string" && endBlock < startBlock) { throw new Error( `Validation failed: Start block for '${source.name}' is after end block (${startBlock} > ${endBlock}).`, ); } + + if (typeof endBlock === "string" && endBlock !== "realtime") { + throw new Error( + `Validation failed: End block for '${source.name}' is ${endBlock}. Expected number or "realtime"`, + ); + } } const network = networks.find((n) => n.name === source.network); @@ -475,7 +457,7 @@ export async function buildConfigAndIndexingFunctions({ topic2: topics.topic2, topic3: topics.topic3, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultLogFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -500,7 +482,7 @@ export async function buildConfigAndIndexingFunctions({ functionSelector: registeredFunctionSelectors, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTraceFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -552,7 +534,7 @@ export async function buildConfigAndIndexingFunctions({ topic2: topics.topic2, topic3: topics.topic3, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultLogFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -581,7 +563,7 @@ export async function buildConfigAndIndexingFunctions({ functionSelector: registeredFunctionSelectors, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTraceFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -649,7 +631,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: logFactory, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTransactionFilterInclude, }, } satisfies AccountSource, @@ -664,7 +646,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: undefined, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTransactionFilterInclude, }, } satisfies AccountSource, @@ -679,7 +661,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: logFactory, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTransferFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -698,7 +680,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: undefined, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTransferFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -748,7 +730,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: validatedAddress, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTransactionFilterInclude, }, } satisfies AccountSource, @@ -763,7 +745,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: undefined, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTransactionFilterInclude, }, } satisfies AccountSource, @@ -778,7 +760,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: validatedAddress, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTransferFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -797,7 +779,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: undefined, includeReverted: false, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultTransferFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -856,9 +838,9 @@ export async function buildConfigAndIndexingFunctions({ type: "block", chainId: network.chainId, interval: interval, - offset: (fromBlock ?? 0) % interval, + offset: fromBlock % interval, fromBlock, - toBlock, + toBlock: toBlock === "realtime" ? undefined : toBlock, include: defaultBlockFilterInclude, }, }) satisfies BlockSource, diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 0c68b8387..1ed5ebb81 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -63,9 +63,11 @@ type DatabaseConfig = type BlockConfig = { /** Block intervals with startBlock (inclusive) and endBlock (inclusive). If `undefined`, events will be processed from block 0 and in real-time. */ - blocks?: [number | undefined, number | undefined][]; + blocks?: BlockRange[]; }; +export type BlockRange = [number, number | "realtime"]; + type TransactionReceiptConfig = { includeTransactionReceipts?: boolean; }; diff --git a/packages/core/src/sync/index.ts b/packages/core/src/sync/index.ts index 0d1a406c7..3bfc66189 100644 --- a/packages/core/src/sync/index.ts +++ b/packages/core/src/sync/index.ts @@ -920,7 +920,7 @@ export const syncDiagnostic = async ({ requestQueue: RequestQueue; }) => { /** Earliest `startBlock` among all `filters` */ - const start = Math.min(...sources.map(({ filter }) => filter.fromBlock ?? 0)); + const start = Math.min(...sources.map(({ filter }) => filter.fromBlock)); /** * Latest `endBlock` among all filters. `undefined` if at least one * of the filters doesn't have an `endBlock`. @@ -983,7 +983,7 @@ export const getCachedBlock = ({ }): Promise | undefined => { const latestCompletedBlocks = sources.map(({ filter }) => { const requiredInterval = [ - filter.fromBlock ?? 0, + filter.fromBlock, filter.toBlock ?? Number.POSITIVE_INFINITY, ] satisfies Interval; const cachedIntervals = historicalSync.intervalsCache.get(filter)!; @@ -995,7 +995,7 @@ export const getCachedBlock = ({ if (completedIntervals.length === 0) return undefined; const earliestCompletedInterval = completedIntervals[0]!; - if (earliestCompletedInterval[0] !== (filter.fromBlock ?? 0)) { + if (earliestCompletedInterval[0] !== filter.fromBlock) { return undefined; } return earliestCompletedInterval[1]; @@ -1014,8 +1014,7 @@ export const getCachedBlock = ({ if ( latestCompletedBlocks.every( (block, i) => - block !== undefined || - (sources[i]!.filter.fromBlock ?? 0) > minCompletedBlock, + block !== undefined || sources[i]!.filter.fromBlock > minCompletedBlock, ) ) { return _eth_getBlockByNumber(requestQueue, { @@ -1084,7 +1083,7 @@ export async function* localHistoricalSyncGenerator({ intervalDifference( [ [ - filter.fromBlock ?? 0, + filter.fromBlock, Math.min( filter.toBlock ?? Number.POSITIVE_INFINITY, totalInterval[1], diff --git a/packages/core/src/sync/source.ts b/packages/core/src/sync/source.ts index ed067d4bd..5560428da 100644 --- a/packages/core/src/sync/source.ts +++ b/packages/core/src/sync/source.ts @@ -71,7 +71,7 @@ export type LogFilter< topic1: LogTopic | undefined; topic2: LogTopic | undefined; topic3: LogTopic | undefined; - fromBlock: number | undefined; + fromBlock: number; toBlock: number | undefined; include: | ( @@ -88,7 +88,7 @@ export type BlockFilter = { chainId: number; interval: number; offset: number; - fromBlock: number | undefined; + fromBlock: number; toBlock: number | undefined; include: `block.${keyof Block}`[] | undefined; }; @@ -106,7 +106,7 @@ export type TransferFilter< ? fromFactory : Address | Address[] | undefined; includeReverted: boolean; - fromBlock: number | undefined; + fromBlock: number; toBlock: number | undefined; include: | ( @@ -131,7 +131,7 @@ export type TransactionFilter< ? toFactory : Address | Address[] | undefined; includeReverted: boolean; - fromBlock: number | undefined; + fromBlock: number; toBlock: number | undefined; include: | ( @@ -157,7 +157,7 @@ export type TraceFilter< functionSelector: Hex | Hex[] | undefined; callType: Trace["result"]["type"] | undefined; includeReverted: boolean; - fromBlock: number | undefined; + fromBlock: number; toBlock: number | undefined; include: | ( From 9d3cc04924b9e61d7405c6c4806e3b3d89b5c857 Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Sat, 4 Jan 2025 23:09:36 +0500 Subject: [PATCH 03/10] resolving fromBlock, toBlock to number --- .../src/build/configAndIndexingFunctions.ts | 48 +++++++-------- packages/core/src/sync-historical/index.ts | 9 ++- packages/core/src/sync-realtime/bloom.ts | 4 +- packages/core/src/sync-realtime/filter.ts | 20 +++---- packages/core/src/sync-store/index.ts | 40 ++++--------- packages/core/src/sync/index.ts | 59 ++++++------------- packages/core/src/sync/source.ts | 10 ++-- 7 files changed, 73 insertions(+), 117 deletions(-) diff --git a/packages/core/src/build/configAndIndexingFunctions.ts b/packages/core/src/build/configAndIndexingFunctions.ts index 83b569d5a..2ff5eea59 100644 --- a/packages/core/src/build/configAndIndexingFunctions.ts +++ b/packages/core/src/build/configAndIndexingFunctions.ts @@ -63,18 +63,22 @@ const flattenSources = < ); }; -function resolveBlockRanges(blocks: BlockRange[] | undefined): BlockRange[] { - const blockRanges: BlockRange[] = +function resolveBlockRanges( + blocks: BlockRange[] | undefined, +): [number, number][] { + const blockRanges: [number, number][] = blocks === undefined || blocks.length === 0 - ? [[0, "realtime"]] + ? [[0, Number.MAX_SAFE_INTEGER]] : blocks.map(([rawStartBlock, rawEndBlock]) => [ Number.isNaN(rawStartBlock) ? 0 : rawStartBlock, - Number.isNaN(rawEndBlock) ? "realtime" : rawEndBlock, + Number.isNaN(rawEndBlock) + ? Number.MAX_SAFE_INTEGER + : (rawEndBlock as number), ]); blockRanges.sort((a, b) => a[0] - b[0]); - const resolvedBlockRanges: BlockRange[] = []; + const resolvedBlockRanges: [number, number][] = []; for (const [curStartBlock, curEndBlock] of blockRanges) { if (resolvedBlockRanges.length === 0) { @@ -84,16 +88,14 @@ function resolveBlockRanges(blocks: BlockRange[] | undefined): BlockRange[] { const last = resolvedBlockRanges[resolvedBlockRanges.length - 1]!; const [lastStartBlock, lastEndBlock] = last; - if (lastEndBlock === "realtime") { + if (lastEndBlock === Number.MAX_SAFE_INTEGER) { break; } // Check for overlapping block ranges - if (curStartBlock === undefined || curStartBlock <= lastEndBlock) { + if (curStartBlock <= lastEndBlock) { resolvedBlockRanges[resolvedBlockRanges.length - 1] = [ lastStartBlock, - curEndBlock === "realtime" || curEndBlock >= lastEndBlock - ? curEndBlock - : lastEndBlock, + curEndBlock >= lastEndBlock ? curEndBlock : lastEndBlock, ]; } else { resolvedBlockRanges.push([curStartBlock, curEndBlock]); @@ -457,7 +459,7 @@ export async function buildConfigAndIndexingFunctions({ topic2: topics.topic2, topic3: topics.topic3, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultLogFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -482,7 +484,7 @@ export async function buildConfigAndIndexingFunctions({ functionSelector: registeredFunctionSelectors, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTraceFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -534,7 +536,7 @@ export async function buildConfigAndIndexingFunctions({ topic2: topics.topic2, topic3: topics.topic3, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultLogFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -563,7 +565,7 @@ export async function buildConfigAndIndexingFunctions({ functionSelector: registeredFunctionSelectors, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTraceFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -631,7 +633,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: logFactory, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTransactionFilterInclude, }, } satisfies AccountSource, @@ -646,7 +648,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: undefined, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTransactionFilterInclude, }, } satisfies AccountSource, @@ -661,7 +663,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: logFactory, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTransferFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -680,7 +682,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: undefined, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTransferFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -730,7 +732,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: validatedAddress, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTransactionFilterInclude, }, } satisfies AccountSource, @@ -745,7 +747,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: undefined, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTransactionFilterInclude, }, } satisfies AccountSource, @@ -760,7 +762,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: validatedAddress, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTransferFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -779,7 +781,7 @@ export async function buildConfigAndIndexingFunctions({ toAddress: undefined, includeReverted: false, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultTransferFilterInclude.concat( source.includeTransactionReceipts ? defaultTransactionReceiptInclude @@ -840,7 +842,7 @@ export async function buildConfigAndIndexingFunctions({ interval: interval, offset: fromBlock % interval, fromBlock, - toBlock: toBlock === "realtime" ? undefined : toBlock, + toBlock, include: defaultBlockFilterInclude, }, }) satisfies BlockSource, diff --git a/packages/core/src/sync-historical/index.ts b/packages/core/src/sync-historical/index.ts index 8badec2c9..0373ab810 100644 --- a/packages/core/src/sync-historical/index.ts +++ b/packages/core/src/sync-historical/index.ts @@ -604,15 +604,14 @@ export const createHistoricalSync = async ( // Skip sync if the interval is after the `toBlock` or before // the `fromBlock`. if ( - (filter.fromBlock !== undefined && - filter.fromBlock > _interval[1]) || - (filter.toBlock !== undefined && filter.toBlock < _interval[0]) + filter.fromBlock > _interval[1] || + filter.toBlock < _interval[0] ) { return; } const interval: Interval = [ - Math.max(filter.fromBlock ?? 0, _interval[0]), - Math.min(filter.toBlock ?? Number.POSITIVE_INFINITY, _interval[1]), + Math.max(filter.fromBlock, _interval[0]), + Math.min(filter.toBlock, _interval[1]), ]; const completedIntervals = intervalsCache.get(filter)!; const requiredIntervals = intervalDifference( diff --git a/packages/core/src/sync-realtime/bloom.ts b/packages/core/src/sync-realtime/bloom.ts index d154e99a3..a4a978695 100644 --- a/packages/core/src/sync-realtime/bloom.ts +++ b/packages/core/src/sync-realtime/bloom.ts @@ -42,8 +42,8 @@ export function isFilterInBloom({ }): boolean { // Return `false` for out of range blocks if ( - hexToNumber(block.number) < (filter.fromBlock ?? 0) || - hexToNumber(block.number) > (filter.toBlock ?? Number.POSITIVE_INFINITY) + hexToNumber(block.number) < filter.fromBlock || + hexToNumber(block.number) > filter.toBlock ) { return false; } diff --git a/packages/core/src/sync-realtime/filter.ts b/packages/core/src/sync-realtime/filter.ts index 3c797f89c..ff6f2923d 100644 --- a/packages/core/src/sync-realtime/filter.ts +++ b/packages/core/src/sync-realtime/filter.ts @@ -84,8 +84,8 @@ export const isLogFilterMatched = ({ }): boolean => { // Return `false` for out of range blocks if ( - hexToNumber(block.number) < (filter.fromBlock ?? 0) || - hexToNumber(block.number) > (filter.toBlock ?? Number.POSITIVE_INFINITY) + hexToNumber(block.number) < filter.fromBlock || + hexToNumber(block.number) > filter.toBlock ) { return false; } @@ -136,8 +136,8 @@ export const isTransactionFilterMatched = ({ }): boolean => { // Return `false` for out of range blocks if ( - hexToNumber(block.number) < (filter.fromBlock ?? 0) || - hexToNumber(block.number) > (filter.toBlock ?? Number.POSITIVE_INFINITY) + hexToNumber(block.number) < filter.fromBlock || + hexToNumber(block.number) > filter.toBlock ) { return false; } @@ -220,8 +220,8 @@ export const isTraceFilterMatched = ({ }): boolean => { // Return `false` for out of range blocks if ( - hexToNumber(block.number) < (filter.fromBlock ?? 0) || - hexToNumber(block.number) > (filter.toBlock ?? Number.POSITIVE_INFINITY) + hexToNumber(block.number) < filter.fromBlock || + hexToNumber(block.number) > filter.toBlock ) { return false; } @@ -305,8 +305,8 @@ export const isTransferFilterMatched = ({ }): boolean => { // Return `false` for out of range blocks if ( - hexToNumber(block.number) < (filter.fromBlock ?? 0) || - hexToNumber(block.number) > (filter.toBlock ?? Number.POSITIVE_INFINITY) + hexToNumber(block.number) < filter.fromBlock || + hexToNumber(block.number) > filter.toBlock ) { return false; } @@ -382,8 +382,8 @@ export const isBlockFilterMatched = ({ }): boolean => { // Return `false` for out of range blocks if ( - hexToNumber(block.number) < (filter.fromBlock ?? 0) || - hexToNumber(block.number) > (filter.toBlock ?? Number.POSITIVE_INFINITY) + hexToNumber(block.number) < filter.fromBlock || + hexToNumber(block.number) > filter.toBlock ) { return false; } diff --git a/packages/core/src/sync-store/index.ts b/packages/core/src/sync-store/index.ts index e646b354a..9a945b21c 100644 --- a/packages/core/src/sync-store/index.ts +++ b/packages/core/src/sync-store/index.ts @@ -551,12 +551,8 @@ export const createSyncStore = ({ return qb; }) .$call((qb) => addressSQL(qb as any, filter.address, "address")) - .$if(filter.fromBlock !== undefined, (qb) => - qb.where("blockNumber", ">=", filter.fromBlock!.toString()), - ) - .$if(filter.toBlock !== undefined, (qb) => - qb.where("blockNumber", "<=", filter.toBlock!.toString()), - ); + .where("blockNumber", ">=", filter.fromBlock.toString()) + .where("blockNumber", "<=", filter.toBlock.toString()); const blockSQL = ( filter: BlockFilter, @@ -578,12 +574,8 @@ export const createSyncStore = ({ .$if(filter !== undefined && filter.interval !== undefined, (qb) => qb.where(ksql`(number - ${filter.offset}) % ${filter.interval} = 0`), ) - .$if(filter.fromBlock !== undefined, (qb) => - qb.where("number", ">=", filter.fromBlock!.toString()), - ) - .$if(filter.toBlock !== undefined, (qb) => - qb.where("number", "<=", filter.toBlock!.toString()), - ); + .where("number", ">=", filter.fromBlock.toString()) + .where("number", "<=", filter.toBlock.toString()); const transactionSQL = ( filter: TransactionFilter, @@ -618,12 +610,8 @@ export const createSyncStore = ({ "0x1", ), ) - .$if(filter.fromBlock !== undefined, (qb) => - qb.where("blockNumber", ">=", filter.fromBlock!.toString()), - ) - .$if(filter.toBlock !== undefined, (qb) => - qb.where("blockNumber", "<=", filter.toBlock!.toString()), - ); + .where("blockNumber", ">=", filter.fromBlock.toString()) + .where("blockNumber", "<=", filter.toBlock.toString()); const transferSQL = ( filter: TransferFilter, @@ -648,12 +636,8 @@ export const createSyncStore = ({ .$if(filter.includeReverted === false, (qb) => qb.where("isReverted", "=", 0), ) - .$if(filter.fromBlock !== undefined, (qb) => - qb.where("blockNumber", ">=", filter.fromBlock!.toString()), - ) - .$if(filter.toBlock !== undefined, (qb) => - qb.where("blockNumber", "<=", filter.toBlock!.toString()), - ); + .where("blockNumber", ">=", filter.fromBlock.toString()) + .where("blockNumber", "<=", filter.toBlock.toString()); const traceSQL = ( filter: TraceFilter, @@ -687,12 +671,8 @@ export const createSyncStore = ({ return qb.where("functionSelector", "=", filter.functionSelector!); } }) - .$if(filter.fromBlock !== undefined, (qb) => - qb.where("blockNumber", ">=", filter.fromBlock!.toString()), - ) - .$if(filter.toBlock !== undefined, (qb) => - qb.where("blockNumber", "<=", filter.toBlock!.toString()), - ); + .where("blockNumber", ">=", filter.fromBlock.toString()) + .where("blockNumber", "<=", filter.toBlock.toString()); const rows = await db.wrap( { diff --git a/packages/core/src/sync/index.ts b/packages/core/src/sync/index.ts index 3bfc66189..8ba4848a7 100644 --- a/packages/core/src/sync/index.ts +++ b/packages/core/src/sync/index.ts @@ -80,7 +80,7 @@ export type Status = { export type SyncProgress = { start: SyncBlock | LightBlock; - end: SyncBlock | LightBlock | undefined; + end: SyncBlock | LightBlock; cached: SyncBlock | LightBlock | undefined; current: SyncBlock | LightBlock | undefined; finalized: SyncBlock | LightBlock; @@ -117,7 +117,7 @@ export const blockToCheckpoint = ( * sync progress has reached the final end block. */ const isSyncEnd = (syncProgress: SyncProgress) => { - if (syncProgress.end === undefined || syncProgress.current === undefined) { + if (syncProgress.current === undefined) { return false; } @@ -143,12 +143,10 @@ const isSyncFinalized = (syncProgress: SyncProgress) => { const getHistoricalLast = ( syncProgress: Pick, ) => { - return syncProgress.end === undefined + return hexToNumber(syncProgress.end.number) > + hexToNumber(syncProgress.finalized.number) ? syncProgress.finalized - : hexToNumber(syncProgress.end.number) > - hexToNumber(syncProgress.finalized.number) - ? syncProgress.finalized - : syncProgress.end; + : syncProgress.end; }; /** Compute the minimum checkpoint, filtering out undefined */ @@ -187,10 +185,6 @@ export const getChainCheckpoint = ({ network: Network; tag: "start" | "current" | "finalized" | "end"; }): string | undefined => { - if (tag === "end" && syncProgress.end === undefined) { - return undefined; - } - if (tag === "current" && isSyncEnd(syncProgress)) { return undefined; } @@ -345,10 +339,6 @@ export const createSync = async (args: CreateSyncParameters): Promise => { getChainCheckpoint({ syncProgress, network, tag }), ); - if (tag === "end" && checkpoints.some((c) => c === undefined)) { - return undefined; - } - if (tag === "current" && checkpoints.every((c) => c === undefined)) { return undefined; } @@ -921,13 +911,8 @@ export const syncDiagnostic = async ({ }) => { /** Earliest `startBlock` among all `filters` */ const start = Math.min(...sources.map(({ filter }) => filter.fromBlock)); - /** - * Latest `endBlock` among all filters. `undefined` if at least one - * of the filters doesn't have an `endBlock`. - */ - const end = sources.some(({ filter }) => filter.toBlock === undefined) - ? undefined - : Math.max(...sources.map(({ filter }) => filter.toBlock!)); + /** Latest `endBlock` among all `filters`. */ + const end = Math.max(...sources.map(({ filter }) => filter.toBlock)); const [remoteChainId, startBlock, latestBlock] = await Promise.all([ requestQueue.request({ method: "eth_chainId" }), @@ -936,16 +921,14 @@ export const syncDiagnostic = async ({ ]); const endBlock = - end === undefined - ? undefined - : end > hexToBigInt(latestBlock.number) - ? ({ - number: toHex(end), - hash: "0x", - parentHash: "0x", - timestamp: toHex(maxCheckpoint.blockTimestamp), - } as LightBlock) - : await _eth_getBlockByNumber(requestQueue, { blockNumber: end }); + end > hexToBigInt(latestBlock.number) + ? ({ + number: toHex(end), + hash: "0x", + parentHash: "0x", + timestamp: toHex(maxCheckpoint.blockTimestamp), + } as LightBlock) + : await _eth_getBlockByNumber(requestQueue, { blockNumber: end }); // Warn if the config has a different chainId than the remote. if (hexToNumber(remoteChainId) !== network.chainId) { @@ -984,7 +967,7 @@ export const getCachedBlock = ({ const latestCompletedBlocks = sources.map(({ filter }) => { const requiredInterval = [ filter.fromBlock, - filter.toBlock ?? Number.POSITIVE_INFINITY, + filter.toBlock, ] satisfies Interval; const cachedIntervals = historicalSync.intervalsCache.get(filter)!; @@ -1081,15 +1064,7 @@ export async function* localHistoricalSyncGenerator({ historicalSync.intervalsCache.entries(), ).flatMap(([filter, interval]) => intervalDifference( - [ - [ - filter.fromBlock, - Math.min( - filter.toBlock ?? Number.POSITIVE_INFINITY, - totalInterval[1], - ), - ], - ], + [[filter.fromBlock, Math.min(filter.toBlock, totalInterval[1])]], interval, ), ); diff --git a/packages/core/src/sync/source.ts b/packages/core/src/sync/source.ts index 5560428da..0f2f48daa 100644 --- a/packages/core/src/sync/source.ts +++ b/packages/core/src/sync/source.ts @@ -72,7 +72,7 @@ export type LogFilter< topic2: LogTopic | undefined; topic3: LogTopic | undefined; fromBlock: number; - toBlock: number | undefined; + toBlock: number; include: | ( | `block.${keyof Block}` @@ -89,7 +89,7 @@ export type BlockFilter = { interval: number; offset: number; fromBlock: number; - toBlock: number | undefined; + toBlock: number; include: `block.${keyof Block}`[] | undefined; }; @@ -107,7 +107,7 @@ export type TransferFilter< : Address | Address[] | undefined; includeReverted: boolean; fromBlock: number; - toBlock: number | undefined; + toBlock: number; include: | ( | `block.${keyof Block}` @@ -132,7 +132,7 @@ export type TransactionFilter< : Address | Address[] | undefined; includeReverted: boolean; fromBlock: number; - toBlock: number | undefined; + toBlock: number; include: | ( | `block.${keyof Block}` @@ -158,7 +158,7 @@ export type TraceFilter< callType: Trace["result"]["type"] | undefined; includeReverted: boolean; fromBlock: number; - toBlock: number | undefined; + toBlock: number; include: | ( | `block.${keyof Block}` From dd0c296423bd4cc1e694ff5538ca80a2fe6468fc Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Thu, 9 Jan 2025 11:25:55 +0500 Subject: [PATCH 04/10] ref: block range simplified --- .../src/build/configAndIndexingFunctions.ts | 66 ++++++++----------- packages/core/src/config/index.ts | 4 +- 2 files changed, 28 insertions(+), 42 deletions(-) diff --git a/packages/core/src/build/configAndIndexingFunctions.ts b/packages/core/src/build/configAndIndexingFunctions.ts index dcfb7a83a..386913734 100644 --- a/packages/core/src/build/configAndIndexingFunctions.ts +++ b/packages/core/src/build/configAndIndexingFunctions.ts @@ -1,5 +1,5 @@ import { BuildError } from "@/common/errors.js"; -import type { BlockRange, Config } from "@/config/index.js"; +import type { Config } from "@/config/index.js"; import { getFinalityBlockCount, getRpcUrlsForClient, @@ -19,6 +19,7 @@ import { defaultTransferFilterInclude, } from "@/sync/source.js"; import { chains } from "@/utils/chains.js"; +import { type Interval, intervalUnion } from "@/utils/interval.js"; import { toLowerCase } from "@/utils/lowercase.js"; import type { Address, Hex, LogTopic } from "viem"; import { buildLogFactory } from "./factory.js"; @@ -28,6 +29,8 @@ import type { RawIndexingFunctions, } from "./index.js"; +type BlockRange = [number, number | "realtime"]; + const flattenSources = < T extends Config["contracts"] | Config["accounts"] | Config["blocks"], >( @@ -59,45 +62,25 @@ const flattenSources = < }; function resolveBlockRanges( - blocks: BlockRange[] | undefined, -): [number, number][] { - const blockRanges: [number, number][] = + blocks: BlockRange[] | BlockRange | undefined, +): Interval[] { + const rawBlockRanges: BlockRange[] = blocks === undefined || blocks.length === 0 - ? [[0, Number.MAX_SAFE_INTEGER]] - : blocks.map(([rawStartBlock, rawEndBlock]) => [ - Number.isNaN(rawStartBlock) ? 0 : rawStartBlock, - Number.isNaN(rawEndBlock) - ? Number.MAX_SAFE_INTEGER - : (rawEndBlock as number), - ]); - - blockRanges.sort((a, b) => a[0] - b[0]); - - const resolvedBlockRanges: [number, number][] = []; - - for (const [curStartBlock, curEndBlock] of blockRanges) { - if (resolvedBlockRanges.length === 0) { - resolvedBlockRanges.push([curStartBlock, curEndBlock]); - continue; - } - - const last = resolvedBlockRanges[resolvedBlockRanges.length - 1]!; - const [lastStartBlock, lastEndBlock] = last; - if (lastEndBlock === Number.MAX_SAFE_INTEGER) { - break; - } - // Check for overlapping block ranges - if (curStartBlock <= lastEndBlock) { - resolvedBlockRanges[resolvedBlockRanges.length - 1] = [ - lastStartBlock, - curEndBlock >= lastEndBlock ? curEndBlock : lastEndBlock, - ]; - } else { - resolvedBlockRanges.push([curStartBlock, curEndBlock]); - } - } + ? [[0, "realtime"]] + : blocks.every((b) => Array.isArray(b)) + ? blocks + : [blocks]; + + const blockRanges: Interval[] = rawBlockRanges.map( + ([rawStartBlock, rawEndBlock]) => [ + Number.isNaN(rawStartBlock) ? 0 : rawStartBlock, + Number.isNaN(rawEndBlock) + ? Number.MAX_SAFE_INTEGER + : (rawEndBlock as number), + ], + ); - return resolvedBlockRanges; + return intervalUnion(blockRanges); } export async function buildConfigAndIndexingFunctions({ @@ -259,7 +242,12 @@ export async function buildConfigAndIndexingFunctions({ } const blockRanges: BlockRange[] = - source.blocks === undefined ? [[0, "realtime"]] : source.blocks; + source.blocks === undefined + ? [[0, "realtime"]] + : source.blocks.every((b) => Array.isArray(b)) + ? source.blocks + : [source.blocks]; + for (const [rawStartBlock, rawEndBlock] of blockRanges) { const startBlock = Number.isNaN(rawStartBlock) ? 0 : rawStartBlock; const endBlock = Number.isNaN(rawEndBlock) ? "realtime" : rawEndBlock; diff --git a/packages/core/src/config/index.ts b/packages/core/src/config/index.ts index 934e6d576..631bdc32b 100644 --- a/packages/core/src/config/index.ts +++ b/packages/core/src/config/index.ts @@ -62,11 +62,9 @@ type DatabaseConfig = type BlockConfig = { /** Block intervals with startBlock (inclusive) and endBlock (inclusive). If `undefined`, events will be processed from block 0 and in real-time. */ - blocks?: BlockRange[]; + blocks?: [number, number | "realtime"][] | [number, number | "realtime"]; }; -export type BlockRange = [number, number | "realtime"]; - type TransactionReceiptConfig = { includeTransactionReceipts?: boolean; }; From afa9f159f45a3f01f1c83a9d64294d38d7c74bdd Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Thu, 9 Jan 2025 12:17:38 +0500 Subject: [PATCH 05/10] build tests --- .../build/configAndIndexingFunctions.test.ts | 90 ++++++++++++++----- 1 file changed, 66 insertions(+), 24 deletions(-) diff --git a/packages/core/src/build/configAndIndexingFunctions.test.ts b/packages/core/src/build/configAndIndexingFunctions.test.ts index 77cc5c0a9..0b739e9af 100644 --- a/packages/core/src/build/configAndIndexingFunctions.test.ts +++ b/packages/core/src/build/configAndIndexingFunctions.test.ts @@ -42,8 +42,7 @@ test("buildConfigAndIndexingFunctions() builds topics for multiple events", asyn network: { mainnet: {} }, abi: [event0, event1], address: address1, - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], }, }, }); @@ -72,8 +71,7 @@ test("buildConfigAndIndexingFunctions() handles overloaded event signatures and network: { mainnet: {} }, abi: [event1, event1Overloaded], address: address1, - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], }, }, }); @@ -102,8 +100,7 @@ test("buildConfigAndIndexingFunctions() handles multiple addresses", async () => network: { mainnet: { address: [address1, address3], - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], }, }, abi: [event1, event1Overloaded], @@ -163,8 +160,7 @@ test("buildConfigAndIndexingFunctions() builds topics for event filter", async ( }, }, address: address1, - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], }, }, }); @@ -203,8 +199,7 @@ test("buildConfigAndIndexingFunctions() builds topics for multiple event filters }, ], address: address1, - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], }, }, }); @@ -237,8 +232,7 @@ test("buildConfigAndIndexingFunctions() overrides default values with network-sp a: { abi: [event0], address: address1, - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], network: { mainnet: { address: address2, @@ -266,8 +260,7 @@ test("buildConfigAndIndexingFunctions() handles network name shortcut", async () network: "mainnet", abi: [event0], address: address1, - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], }, }, }); @@ -441,7 +434,7 @@ test("buildConfigAndIndexingFunctions() validates address length", async () => { ); }); -test("buildConfigAndIndexingFunctions() coerces NaN startBlock to undefined", async () => { +test("buildConfigAndIndexingFunctions() coerces NaN startBlock to 0", async () => { const config = createConfig({ networks: { mainnet: { chainId: 1, transport: http("http://127.0.0.1:8545") }, @@ -450,7 +443,7 @@ test("buildConfigAndIndexingFunctions() coerces NaN startBlock to undefined", as a: { network: { mainnet: {} }, abi: [event0, event1], - startBlock: Number.NaN, + blocks: [Number.NaN, 16370020], }, }, }); @@ -460,7 +453,7 @@ test("buildConfigAndIndexingFunctions() coerces NaN startBlock to undefined", as rawIndexingFunctions: [{ name: "a:Event0", fn: () => {} }], }); - expect(sources[0]?.filter.fromBlock).toBe(undefined); + expect(sources[0]?.filter.fromBlock).toBe(0); }); test("buildConfigAndIndexingFunctions() includeTransactionReceipts", async () => { @@ -565,7 +558,7 @@ test("buildConfigAndIndexingFunctions() includeCallTraces with factory", async ( expect(shouldGetTransactionReceipt(sources[0]!.filter)).toBe(false); }); -test("buildConfigAndIndexingFunctions() coerces NaN endBlock to undefined", async () => { +test("buildConfigAndIndexingFunctions() coerces NaN endBlock to Number.MAX_SAFE_INTEGER", async () => { const config = createConfig({ networks: { mainnet: { chainId: 1, transport: http("http://127.0.0.1:8545") }, @@ -574,7 +567,7 @@ test("buildConfigAndIndexingFunctions() coerces NaN endBlock to undefined", asyn a: { network: { mainnet: {} }, abi: [event0, event1], - endBlock: Number.NaN, + blocks: [16370000, Number.NaN], }, }, }); @@ -584,7 +577,58 @@ test("buildConfigAndIndexingFunctions() coerces NaN endBlock to undefined", asyn rawIndexingFunctions: [{ name: "a:Event0", fn: () => {} }], }); - expect(sources[0]!.filter.toBlock).toBe(undefined); + expect(sources[0]!.filter.toBlock).toBe(Number.MAX_SAFE_INTEGER); +}); + +test("buildConfigAndIndexingFunctions() resolves overlapping block ranges", async () => { + const config = createConfig({ + networks: { + mainnet: { chainId: 1, transport: http("http://127.0.0.1:8545") }, + }, + contracts: { + a: { + network: { mainnet: {} }, + abi: [event0, event1], + blocks: [ + [16370000, 16370020], + [16370010, 16370030], + ], + }, + }, + }); + + const { sources } = await buildConfigAndIndexingFunctions({ + config, + rawIndexingFunctions: [{ name: "a:Event0", fn: () => {} }], + }); + + expect(sources[0]!.filter.fromBlock).toBe(16370000); + expect(sources[0]!.filter.toBlock).toBe(16370030); +}); + +test("buildConfigAndIndexingFunctions() multiple block ranges", async () => { + const config = createConfig({ + networks: { + mainnet: { chainId: 1, transport: http("http://127.0.0.1:8545") }, + }, + contracts: { + a: { + network: { mainnet: {} }, + abi: [event0, event1], + blocks: [ + [16370000, 16370020], + [16370040, 16370060], + ], + }, + }, + }); + + const { sources } = await buildConfigAndIndexingFunctions({ + config, + rawIndexingFunctions: [{ name: "a:Event0", fn: () => {} }], + }); + + expect(sources).toHaveLength(2); }); test("buildConfigAndIndexingFunctions() account source", async () => { @@ -596,8 +640,7 @@ test("buildConfigAndIndexingFunctions() account source", async () => { a: { network: { mainnet: {} }, address: address1, - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], }, }, }); @@ -636,8 +679,7 @@ test("buildConfigAndIndexingFunctions() block source", async () => { blocks: { a: { network: { mainnet: {} }, - startBlock: 16370000, - endBlock: 16370020, + blocks: [16370000, 16370020], }, }, }); From c3f7934b190fe640268398a95ae9dd51dff6640d Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Thu, 9 Jan 2025 12:31:33 +0500 Subject: [PATCH 06/10] minor fix --- packages/core/src/build/configAndIndexingFunctions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/build/configAndIndexingFunctions.ts b/packages/core/src/build/configAndIndexingFunctions.ts index 3fbf2bbe0..696a1bea4 100644 --- a/packages/core/src/build/configAndIndexingFunctions.ts +++ b/packages/core/src/build/configAndIndexingFunctions.ts @@ -75,7 +75,7 @@ function resolveBlockRanges( const blockRanges: Interval[] = rawBlockRanges.map( ([rawStartBlock, rawEndBlock]) => [ Number.isNaN(rawStartBlock) ? 0 : rawStartBlock, - Number.isNaN(rawEndBlock) + Number.isNaN(rawEndBlock) || rawEndBlock === "realtime" ? Number.MAX_SAFE_INTEGER : (rawEndBlock as number), ], From 36fde439f376a04ab367c6b0e6b9631b8b25c0d6 Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Wed, 15 Jan 2025 22:01:12 +0500 Subject: [PATCH 07/10] build-config: latest from and to Block --- .../src/build/configAndIndexingFunctions.ts | 81 +++++++++++++------ packages/core/src/config/index.ts | 4 +- 2 files changed, 60 insertions(+), 25 deletions(-) diff --git a/packages/core/src/build/configAndIndexingFunctions.ts b/packages/core/src/build/configAndIndexingFunctions.ts index 696a1bea4..8e0385269 100644 --- a/packages/core/src/build/configAndIndexingFunctions.ts +++ b/packages/core/src/build/configAndIndexingFunctions.ts @@ -26,11 +26,12 @@ import { import { chains } from "@/utils/chains.js"; import { type Interval, intervalUnion } from "@/utils/interval.js"; import { toLowerCase } from "@/utils/lowercase.js"; +import { _eth_getBlockByNumber } from "@/utils/rpc.js"; import { dedupe } from "@ponder/common"; -import type { Hex, LogTopic } from "viem"; +import { type Hex, type LogTopic, hexToNumber } from "viem"; import { buildLogFactory } from "./factory.js"; -type BlockRange = [number, number | "realtime"]; +type BlockRange = [number | "latest", number | "realtime" | "latest"]; const flattenSources = < T extends Config["contracts"] | Config["accounts"] | Config["blocks"], @@ -64,13 +65,22 @@ const flattenSources = < function resolveBlockRanges( blocks: BlockRange[] | BlockRange | undefined, + latest: number, ): Interval[] { - const rawBlockRanges: BlockRange[] = + const rawBlockRanges: [number, number | "realtime"][] = blocks === undefined || blocks.length === 0 ? [[0, "realtime"]] : blocks.every((b) => Array.isArray(b)) - ? blocks - : [blocks]; + ? blocks.map(([fromBlock, toBlock]) => [ + fromBlock === "latest" ? latest : fromBlock, + toBlock === "latest" ? latest : toBlock, + ]) + : [ + [ + blocks[0] === "latest" ? latest : blocks[0], + blocks[1] === "latest" ? latest : blocks[1], + ], + ]; const blockRanges: Interval[] = rawBlockRanges.map( ([rawStartBlock, rawEndBlock]) => [ @@ -98,7 +108,9 @@ export async function buildConfigAndIndexingFunctions({ }> { const logs: { level: "warn" | "info" | "debug"; msg: string }[] = []; - const networks: Network[] = await Promise.all( + const latestBlockNumbers = new Map(); + + const networks = await Promise.all( Object.entries(config.networks).map(async ([networkName, network]) => { const { chainId, transport } = network; @@ -128,7 +140,7 @@ export async function buildConfigAndIndexingFunctions({ ); } - return { + const resolvedNetwork = { name: networkName, chainId, chain, @@ -138,6 +150,14 @@ export async function buildConfigAndIndexingFunctions({ finalityBlockCount: getFinalityBlockCount({ chainId }), disableCache: network.disableCache ?? false, } satisfies Network; + + const latest: Hex = await network.transport({ chain }).request({ + method: "eth_blockNumber", + }); + + latestBlockNumbers.set(networkName, hexToNumber(latest)); + + return resolvedNetwork; }), ); @@ -242,12 +262,33 @@ export async function buildConfigAndIndexingFunctions({ ); } - const blockRanges: BlockRange[] = + const network = networks.find((n) => n.name === source.network); + if (!network) { + throw new Error( + `Validation failed: Invalid network for '${ + source.name + }'. Got '${source.network}', expected one of [${networks + .map((n) => `'${n.name}'`) + .join(", ")}].`, + ); + } + + const latest = latestBlockNumbers.get(source.network)!; + + const blockRanges: [number, number | "realtime"][] = source.blocks === undefined ? [[0, "realtime"]] : source.blocks.every((b) => Array.isArray(b)) - ? source.blocks - : [source.blocks]; + ? source.blocks.map(([fromBlock, toBlock]) => [ + fromBlock === "latest" ? latest : fromBlock, + toBlock === "latest" ? latest : toBlock, + ]) + : [ + [ + source.blocks[0] === "latest" ? latest : source.blocks[0], + source.blocks[1] === "latest" ? latest : source.blocks[1], + ], + ]; for (const [rawStartBlock, rawEndBlock] of blockRanges) { const startBlock = Number.isNaN(rawStartBlock) ? 0 : rawStartBlock; @@ -264,17 +305,6 @@ export async function buildConfigAndIndexingFunctions({ ); } } - - const network = networks.find((n) => n.name === source.network); - if (!network) { - throw new Error( - `Validation failed: Invalid network for '${ - source.name - }'. Got '${source.network}', expected one of [${networks - .map((n) => `'${n.name}'`) - .join(", ")}].`, - ); - } } const contractSources: ContractSource[] = flattenSources( @@ -406,7 +436,8 @@ export async function buildConfigAndIndexingFunctions({ }); } - const resolvedBlockRanges = resolveBlockRanges(source.blocks); + const latest = latestBlockNumbers.get(source.network)!; + const resolvedBlockRanges = resolveBlockRanges(source.blocks, latest); const contractMetadata = { type: "contract", @@ -584,7 +615,8 @@ export async function buildConfigAndIndexingFunctions({ .flatMap((source): AccountSource[] => { const network = networks.find((n) => n.name === source.network)!; - const resolvedBlockRanges = resolveBlockRanges(source.blocks); + const latest = latestBlockNumbers.get(source.network)!; + const resolvedBlockRanges = resolveBlockRanges(source.blocks, latest); const resolvedAddress = source?.address; @@ -811,7 +843,8 @@ export async function buildConfigAndIndexingFunctions({ ); } - const resolvedBlockRanges = resolveBlockRanges(source.blocks); + const latest = latestBlockNumbers.get(source.network)!; + const resolvedBlockRanges = resolveBlockRanges(source.blocks, latest); return resolvedBlockRanges.map( ([fromBlock, toBlock]) => diff --git a/packages/core/src/config/index.ts b/packages/core/src/config/index.ts index 631bdc32b..950fadc93 100644 --- a/packages/core/src/config/index.ts +++ b/packages/core/src/config/index.ts @@ -62,7 +62,9 @@ type DatabaseConfig = type BlockConfig = { /** Block intervals with startBlock (inclusive) and endBlock (inclusive). If `undefined`, events will be processed from block 0 and in real-time. */ - blocks?: [number, number | "realtime"][] | [number, number | "realtime"]; + blocks?: + | [number | "latest", number | "realtime" | "latest"][] + | [number | "latest", number | "realtime" | "latest"]; }; type TransactionReceiptConfig = { From 016bb6c1c2df5ab3e0b90a7e230b244756931075 Mon Sep 17 00:00:00 2001 From: Kyle Scott Date: Tue, 21 Jan 2025 14:51:42 -0500 Subject: [PATCH 08/10] fix test type errors --- packages/core/src/config/index.test.ts | 8 +++---- packages/core/src/sync-store/index.test.ts | 28 +++++++++++----------- packages/core/src/types/virtual.test-d.ts | 16 +++++-------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/packages/core/src/config/index.test.ts b/packages/core/src/config/index.test.ts index 05dd94959..56c5373a0 100644 --- a/packages/core/src/config/index.test.ts +++ b/packages/core/src/config/index.test.ts @@ -25,12 +25,12 @@ test("createConfig basic", () => { c1: { abi: [event1], network: "mainnet", - startBlock: 0, + blocks: [0, Number.POSITIVE_INFINITY], }, c2: { abi: [event1], network: "optimism", - startBlock: 0, + blocks: [0, Number.POSITIVE_INFINITY], }, }, }); @@ -185,7 +185,7 @@ test("createConfig network overrides", () => { c1: { abi: [event1], network: "mainnet", - startBlock: 0, + blocks: [0, Number.POSITIVE_INFINITY], }, c2: { abi: [event0, event1], @@ -200,7 +200,7 @@ test("createConfig network overrides", () => { }, }, }, - startBlock: 0, + blocks: [0, Number.POSITIVE_INFINITY], }, }, }); diff --git a/packages/core/src/sync-store/index.test.ts b/packages/core/src/sync-store/index.test.ts index 2d78c753d..a5f358682 100644 --- a/packages/core/src/sync-store/index.test.ts +++ b/packages/core/src/sync-store/index.test.ts @@ -80,8 +80,8 @@ test("getIntervals() empty", async (context) => { chainId: 1, interval: 1, offset: 0, - fromBlock: undefined, - toBlock: undefined, + fromBlock: 0, + toBlock: Number.POSITIVE_INFINITY, include: [], } satisfies BlockFilter; @@ -124,8 +124,8 @@ test("getIntervals() returns intervals", async (context) => { chainId: 1, interval: 1, offset: 0, - fromBlock: undefined, - toBlock: undefined, + fromBlock: 0, + toBlock: Number.POSITIVE_INFINITY, include: [], } satisfies BlockFilter; @@ -183,8 +183,8 @@ test("getIntervals() merges intervals", async (context) => { chainId: 1, interval: 1, offset: 0, - fromBlock: undefined, - toBlock: undefined, + fromBlock: 0, + toBlock: Number.POSITIVE_INFINITY, include: [], } satisfies BlockFilter; @@ -254,8 +254,8 @@ test("getIntervals() adjacent intervals", async (context) => { topic2: null, topic3: null, address: [zeroAddress], - fromBlock: undefined, - toBlock: undefined, + fromBlock: 0, + toBlock: Number.POSITIVE_INFINITY, include: [], } satisfies LogFilter; @@ -332,8 +332,8 @@ test("insertIntervals() merges duplicates", async (context) => { chainId: 1, interval: 1, offset: 0, - fromBlock: undefined, - toBlock: undefined, + fromBlock: 0, + toBlock: Number.POSITIVE_INFINITY, include: [], } satisfies BlockFilter; @@ -408,8 +408,8 @@ test("insertIntervals() preserves fragments", async (context) => { topic2: null, topic3: null, address: [zeroAddress, ALICE], - fromBlock: undefined, - toBlock: undefined, + fromBlock: 0, + toBlock: Number.POSITIVE_INFINITY, include: [], } satisfies LogFilter; @@ -1404,8 +1404,8 @@ test("getEvents() returns events", async (context) => { topic1: null, topic2: null, topic3: null, - fromBlock: undefined, - toBlock: undefined, + fromBlock: 0, + toBlock: Number.POSITIVE_INFINITY, include: [], } satisfies LogFilter; diff --git a/packages/core/src/types/virtual.test-d.ts b/packages/core/src/types/virtual.test-d.ts index bfa28130a..00575b583 100644 --- a/packages/core/src/types/virtual.test-d.ts +++ b/packages/core/src/types/virtual.test-d.ts @@ -55,7 +55,7 @@ const config = createConfig({ abi: [event0, func0], network: "mainnet", address: "0x", - startBlock: 0, + blocks: [0, Number.POSITIVE_INFINITY], includeTransactionReceipts: false, includeCallTraces: true, }, @@ -64,7 +64,7 @@ const config = createConfig({ address: "0x69", network: { mainnet: { - startBlock: 1, + blocks: [1, Number.POSITIVE_INFINITY], includeTransactionReceipts: true, includeCallTraces: true, }, @@ -81,7 +81,7 @@ const config = createConfig({ blocks: { b1: { interval: 2, - startBlock: 1, + blocks: [1, Number.POSITIVE_INFINITY], network: "mainnet", }, }, @@ -283,18 +283,14 @@ test("Context contracts", () => { // ^? type expectedAbi = [Event1, Event1Overloaded, Func1, Func1Overloaded]; - type expectedStartBlock = 1 | undefined; - type expectedEndBlock = undefined; + type expectedBlocks = [1, number] | undefined; type expectedAddress = "0x69"; assertType({} as any as expectedAbi); assertType({} as any as a["abi"]); - assertType({} as any as expectedStartBlock); - assertType({} as any as a["startBlock"]); - - assertType({} as any as expectedEndBlock); - assertType({} as any as a["endBlock"]); + assertType({} as any as expectedBlocks); + assertType({} as any as a["blocks"]); assertType({} as any as expectedAddress); assertType({} as any as a["address"]); From c40ddb03893a941512fbd1d6a54b6731638887b5 Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Wed, 22 Jan 2025 16:16:16 +0500 Subject: [PATCH 09/10] build: tags description, fetch latest when needed only --- .../src/build/configAndIndexingFunctions.ts | 391 ++++++++++-------- packages/core/src/config/index.ts | 9 +- 2 files changed, 228 insertions(+), 172 deletions(-) diff --git a/packages/core/src/build/configAndIndexingFunctions.ts b/packages/core/src/build/configAndIndexingFunctions.ts index 8e0385269..938a8f1f0 100644 --- a/packages/core/src/build/configAndIndexingFunctions.ts +++ b/packages/core/src/build/configAndIndexingFunctions.ts @@ -31,7 +31,15 @@ import { dedupe } from "@ponder/common"; import { type Hex, type LogTopic, hexToNumber } from "viem"; import { buildLogFactory } from "./factory.js"; -type BlockRange = [number | "latest", number | "realtime" | "latest"]; +/** + * Block intervals with startBlock (inclusive) and endBlock (inclusive). + * - startBlock: `number | "latest"` + * - endBlock: `number | "latest" | "realtime"` + * - `number`: A specific block number. + * - `"latest"`: The latest block number at the startup of the Ponder instance. + * - `"realtime"`: Indefinite/live indexing. + */ +type BlockRange = [number | "latest", number | "latest" | "realtime"]; const flattenSources = < T extends Config["contracts"] | Config["accounts"] | Config["blocks"], @@ -63,37 +71,6 @@ const flattenSources = < ); }; -function resolveBlockRanges( - blocks: BlockRange[] | BlockRange | undefined, - latest: number, -): Interval[] { - const rawBlockRanges: [number, number | "realtime"][] = - blocks === undefined || blocks.length === 0 - ? [[0, "realtime"]] - : blocks.every((b) => Array.isArray(b)) - ? blocks.map(([fromBlock, toBlock]) => [ - fromBlock === "latest" ? latest : fromBlock, - toBlock === "latest" ? latest : toBlock, - ]) - : [ - [ - blocks[0] === "latest" ? latest : blocks[0], - blocks[1] === "latest" ? latest : blocks[1], - ], - ]; - - const blockRanges: Interval[] = rawBlockRanges.map( - ([rawStartBlock, rawEndBlock]) => [ - Number.isNaN(rawStartBlock) ? 0 : rawStartBlock, - Number.isNaN(rawEndBlock) || rawEndBlock === "realtime" - ? Number.MAX_SAFE_INTEGER - : (rawEndBlock as number), - ], - ); - - return intervalUnion(blockRanges); -} - export async function buildConfigAndIndexingFunctions({ config, rawIndexingFunctions, @@ -108,7 +85,57 @@ export async function buildConfigAndIndexingFunctions({ }> { const logs: { level: "warn" | "info" | "debug"; msg: string }[] = []; - const latestBlockNumbers = new Map(); + const latestBlockNumbers = new Map>(); + + const latest = async (network: Network) => { + if (latestBlockNumbers.has(network.name)) { + return hexToNumber(await latestBlockNumbers.get(network.name)!); + } + + const latest: Promise = network.transport.request({ + method: "eth_blockNumber", + }); + + latestBlockNumbers.set(network.name, latest); + + return hexToNumber(await latest); + }; + + const resolveBlockRanges = async ( + blocks: BlockRange[] | BlockRange | undefined, + network: Network, + ) => { + let rawBlockRanges: [number, number | "realtime"][]; + + if (blocks === undefined || blocks.length === 0) { + rawBlockRanges = [[0, "realtime"]]; + } else if (blocks.every((b) => Array.isArray(b))) { + rawBlockRanges = await Promise.all( + blocks.map(async ([fromBlock, toBlock]) => [ + fromBlock === "latest" ? await latest(network) : fromBlock, + toBlock === "latest" ? await latest(network) : toBlock, + ]), + ); + } else { + rawBlockRanges = [ + [ + blocks[0] === "latest" ? await latest(network) : blocks[0], + blocks[1] === "latest" ? await latest(network) : blocks[1], + ], + ]; + } + + const blockRanges: Interval[] = rawBlockRanges.map( + ([rawStartBlock, rawEndBlock]) => [ + Number.isNaN(rawStartBlock) ? 0 : rawStartBlock, + Number.isNaN(rawEndBlock) || rawEndBlock === "realtime" + ? Number.MAX_SAFE_INTEGER + : (rawEndBlock as number), + ], + ); + + return intervalUnion(blockRanges); + }; const networks = await Promise.all( Object.entries(config.networks).map(async ([networkName, network]) => { @@ -151,12 +178,6 @@ export async function buildConfigAndIndexingFunctions({ disableCache: network.disableCache ?? false, } satisfies Network; - const latest: Hex = await network.transport({ chain }).request({ - method: "eth_blockNumber", - }); - - latestBlockNumbers.set(networkName, hexToNumber(latest)); - return resolvedNetwork; }), ); @@ -273,20 +294,24 @@ export async function buildConfigAndIndexingFunctions({ ); } - const latest = latestBlockNumbers.get(source.network)!; - const blockRanges: [number, number | "realtime"][] = source.blocks === undefined ? [[0, "realtime"]] : source.blocks.every((b) => Array.isArray(b)) - ? source.blocks.map(([fromBlock, toBlock]) => [ - fromBlock === "latest" ? latest : fromBlock, - toBlock === "latest" ? latest : toBlock, - ]) + ? await Promise.all( + source.blocks.map(async ([fromBlock, toBlock]) => [ + fromBlock === "latest" ? await latest(network) : fromBlock, + toBlock === "latest" ? await latest(network) : toBlock, + ]), + ) : [ [ - source.blocks[0] === "latest" ? latest : source.blocks[0], - source.blocks[1] === "latest" ? latest : source.blocks[1], + source.blocks[0] === "latest" + ? await latest(network) + : source.blocks[0], + source.blocks[1] === "latest" + ? await latest(network) + : source.blocks[1], ], ]; @@ -307,8 +332,19 @@ export async function buildConfigAndIndexingFunctions({ } } - const contractSources: ContractSource[] = flattenSources( - config.contracts ?? {}, + const contractSources: ContractSource[] = ( + await Promise.all( + flattenSources(config.contracts ?? {}).map( + async ({ blocks, network, ...rest }) => ({ + blocks: await resolveBlockRanges( + blocks, + networks.find((n) => n.name === network)!, + ), + network, + ...rest, + }), + ), + ) ) .flatMap((source): ContractSource[] => { const network = networks.find((n) => n.name === source.network)!; @@ -436,9 +472,6 @@ export async function buildConfigAndIndexingFunctions({ }); } - const latest = latestBlockNumbers.get(source.network)!; - const resolvedBlockRanges = resolveBlockRanges(source.blocks, latest); - const contractMetadata = { type: "contract", abi: source.abi, @@ -461,7 +494,7 @@ export async function buildConfigAndIndexingFunctions({ }); const logSources = topicsArray.flatMap((topics) => - resolvedBlockRanges.map( + source.blocks.map( ([fromBlock, toBlock]) => ({ ...contractMetadata, @@ -486,7 +519,7 @@ export async function buildConfigAndIndexingFunctions({ ); if (source.includeCallTraces) { - const callTraceSources = resolvedBlockRanges.map( + const callTraceSources = source.blocks.map( ([fromBlock, toBlock]) => ({ ...contractMetadata, @@ -538,7 +571,7 @@ export async function buildConfigAndIndexingFunctions({ : undefined; const logSources = topicsArray.flatMap((topics) => - resolvedBlockRanges.map( + source.blocks.map( ([fromBlock, toBlock]) => ({ ...contractMetadata, @@ -563,7 +596,7 @@ export async function buildConfigAndIndexingFunctions({ ); if (source.includeCallTraces) { - const callTraceSources = resolvedBlockRanges.map( + const callTraceSources = source.blocks.map( ([fromBlock, toBlock]) => ({ ...contractMetadata, @@ -611,13 +644,23 @@ export async function buildConfigAndIndexingFunctions({ return hasNoRegisteredIndexingFunctions === false; }); - const accountSources: AccountSource[] = flattenSources(config.accounts ?? {}) + const accountSources: AccountSource[] = ( + await Promise.all( + flattenSources(config.accounts ?? {}).map( + async ({ blocks, network, ...rest }) => ({ + blocks: await resolveBlockRanges( + blocks, + networks.find((n) => n.name === network)!, + ), + network, + ...rest, + }), + ), + ) + ) .flatMap((source): AccountSource[] => { const network = networks.find((n) => n.name === source.network)!; - const latest = latestBlockNumbers.get(source.network)!; - const resolvedBlockRanges = resolveBlockRanges(source.blocks, latest); - const resolvedAddress = source?.address; if (resolvedAddress === undefined) { @@ -636,106 +679,7 @@ export async function buildConfigAndIndexingFunctions({ ...resolvedAddress, }); - const accountSources = resolvedBlockRanges.flatMap( - ([fromBlock, toBlock]) => [ - { - type: "account", - name: source.name, - network, - filter: { - type: "transaction", - chainId: network.chainId, - fromAddress: undefined, - toAddress: logFactory, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTransactionFilterInclude, - }, - } satisfies AccountSource, - { - type: "account", - name: source.name, - network, - filter: { - type: "transaction", - chainId: network.chainId, - fromAddress: logFactory, - toAddress: undefined, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTransactionFilterInclude, - }, - } satisfies AccountSource, - { - type: "account", - name: source.name, - network, - filter: { - type: "transfer", - chainId: network.chainId, - fromAddress: undefined, - toAddress: logFactory, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTransferFilterInclude.concat( - source.includeTransactionReceipts - ? defaultTransactionReceiptInclude - : [], - ), - }, - } satisfies AccountSource, - { - type: "account", - name: source.name, - network, - filter: { - type: "transfer", - chainId: network.chainId, - fromAddress: logFactory, - toAddress: undefined, - includeReverted: false, - fromBlock, - toBlock, - include: defaultTransferFilterInclude.concat( - source.includeTransactionReceipts - ? defaultTransactionReceiptInclude - : [], - ), - }, - } satisfies AccountSource, - ], - ); - - return accountSources; - } - - for (const address of Array.isArray(resolvedAddress) - ? resolvedAddress - : [resolvedAddress]) { - if (!address!.startsWith("0x")) - throw new Error( - `Validation failed: Invalid prefix for address '${address}'. Got '${address!.slice( - 0, - 2, - )}', expected '0x'.`, - ); - if (address!.length !== 42) - throw new Error( - `Validation failed: Invalid length for address '${address}'. Got ${address!.length}, expected 42 characters.`, - ); - } - - const validatedAddress = Array.isArray(resolvedAddress) - ? dedupe(resolvedAddress).map((r) => toLowerCase(r)) - : resolvedAddress !== undefined - ? toLowerCase(resolvedAddress) - : undefined; - - const accountSources = resolvedBlockRanges.flatMap( - ([fromBlock, toBlock]) => [ + const accountSources = source.blocks.flatMap(([fromBlock, toBlock]) => [ { type: "account", name: source.name, @@ -744,7 +688,7 @@ export async function buildConfigAndIndexingFunctions({ type: "transaction", chainId: network.chainId, fromAddress: undefined, - toAddress: validatedAddress, + toAddress: logFactory, includeReverted: false, fromBlock, toBlock, @@ -758,7 +702,7 @@ export async function buildConfigAndIndexingFunctions({ filter: { type: "transaction", chainId: network.chainId, - fromAddress: validatedAddress, + fromAddress: logFactory, toAddress: undefined, includeReverted: false, fromBlock, @@ -774,7 +718,7 @@ export async function buildConfigAndIndexingFunctions({ type: "transfer", chainId: network.chainId, fromAddress: undefined, - toAddress: validatedAddress, + toAddress: logFactory, includeReverted: false, fromBlock, toBlock, @@ -792,7 +736,7 @@ export async function buildConfigAndIndexingFunctions({ filter: { type: "transfer", chainId: network.chainId, - fromAddress: validatedAddress, + fromAddress: logFactory, toAddress: undefined, includeReverted: false, fromBlock, @@ -804,8 +748,103 @@ export async function buildConfigAndIndexingFunctions({ ), }, } satisfies AccountSource, - ], - ); + ]); + + return accountSources; + } + + for (const address of Array.isArray(resolvedAddress) + ? resolvedAddress + : [resolvedAddress]) { + if (!address!.startsWith("0x")) + throw new Error( + `Validation failed: Invalid prefix for address '${address}'. Got '${address!.slice( + 0, + 2, + )}', expected '0x'.`, + ); + if (address!.length !== 42) + throw new Error( + `Validation failed: Invalid length for address '${address}'. Got ${address!.length}, expected 42 characters.`, + ); + } + + const validatedAddress = Array.isArray(resolvedAddress) + ? dedupe(resolvedAddress).map((r) => toLowerCase(r)) + : resolvedAddress !== undefined + ? toLowerCase(resolvedAddress) + : undefined; + + const accountSources = source.blocks.flatMap(([fromBlock, toBlock]) => [ + { + type: "account", + name: source.name, + network, + filter: { + type: "transaction", + chainId: network.chainId, + fromAddress: undefined, + toAddress: validatedAddress, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTransactionFilterInclude, + }, + } satisfies AccountSource, + { + type: "account", + name: source.name, + network, + filter: { + type: "transaction", + chainId: network.chainId, + fromAddress: validatedAddress, + toAddress: undefined, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTransactionFilterInclude, + }, + } satisfies AccountSource, + { + type: "account", + name: source.name, + network, + filter: { + type: "transfer", + chainId: network.chainId, + fromAddress: undefined, + toAddress: validatedAddress, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTransferFilterInclude.concat( + source.includeTransactionReceipts + ? defaultTransactionReceiptInclude + : [], + ), + }, + } satisfies AccountSource, + { + type: "account", + name: source.name, + network, + filter: { + type: "transfer", + chainId: network.chainId, + fromAddress: validatedAddress, + toAddress: undefined, + includeReverted: false, + fromBlock, + toBlock, + include: defaultTransferFilterInclude.concat( + source.includeTransactionReceipts + ? defaultTransactionReceiptInclude + : [], + ), + }, + } satisfies AccountSource, + ]); return accountSources; }) @@ -830,7 +869,20 @@ export async function buildConfigAndIndexingFunctions({ return hasRegisteredIndexingFunction; }); - const blockSources: BlockSource[] = flattenSources(config.blocks ?? {}) + const blockSources: BlockSource[] = ( + await Promise.all( + flattenSources(config.blocks ?? {}).map( + async ({ blocks, network, ...rest }) => ({ + blocks: await resolveBlockRanges( + blocks, + networks.find((n) => n.name === network)!, + ), + network, + ...rest, + }), + ), + ) + ) .flatMap((source) => { const network = networks.find((n) => n.name === source.network)!; @@ -843,10 +895,7 @@ export async function buildConfigAndIndexingFunctions({ ); } - const latest = latestBlockNumbers.get(source.network)!; - const resolvedBlockRanges = resolveBlockRanges(source.blocks, latest); - - return resolvedBlockRanges.map( + return source.blocks.map( ([fromBlock, toBlock]) => ({ type: "block", diff --git a/packages/core/src/config/index.ts b/packages/core/src/config/index.ts index 950fadc93..5d849bdc3 100644 --- a/packages/core/src/config/index.ts +++ b/packages/core/src/config/index.ts @@ -61,7 +61,14 @@ type DatabaseConfig = // base type BlockConfig = { - /** Block intervals with startBlock (inclusive) and endBlock (inclusive). If `undefined`, events will be processed from block 0 and in real-time. */ + /** + * Block intervals with startBlock (inclusive) and endBlock (inclusive). + * - startBlock: `number | "latest"` + * - endBlock: `number | "latest" | "realtime"` + * - `number`: A specific block number. + * - `"latest"`: The latest block number at the startup of the Ponder instance. + * - `"realtime"`: Indefinite/live indexing. + */ blocks?: | [number | "latest", number | "realtime" | "latest"][] | [number | "latest", number | "realtime" | "latest"]; From 803e7474fb8ac5f4cc329ffe09a8dddfd960c319 Mon Sep 17 00:00:00 2001 From: Khaidar Kairbek Date: Wed, 22 Jan 2025 16:27:58 +0500 Subject: [PATCH 10/10] sync-store: tests fix --- packages/core/src/sync-store/index.test.ts | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/core/src/sync-store/index.test.ts b/packages/core/src/sync-store/index.test.ts index a5f358682..418c16052 100644 --- a/packages/core/src/sync-store/index.test.ts +++ b/packages/core/src/sync-store/index.test.ts @@ -93,11 +93,11 @@ test("getIntervals() empty", async (context) => { Map { { "chainId": 1, - "fromBlock": undefined, + "fromBlock": 0, "include": [], "interval": 1, "offset": 0, - "toBlock": undefined, + "toBlock": Infinity, "type": "block", } => [ { @@ -147,11 +147,11 @@ test("getIntervals() returns intervals", async (context) => { Map { { "chainId": 1, - "fromBlock": undefined, + "fromBlock": 0, "include": [], "interval": 1, "offset": 0, - "toBlock": undefined, + "toBlock": Infinity, "type": "block", } => [ { @@ -215,11 +215,11 @@ test("getIntervals() merges intervals", async (context) => { Map { { "chainId": 1, - "fromBlock": undefined, + "fromBlock": 0, "include": [], "interval": 1, "offset": 0, - "toBlock": undefined, + "toBlock": Infinity, "type": "block", } => [ { @@ -290,9 +290,9 @@ test("getIntervals() adjacent intervals", async (context) => { "0x0000000000000000000000000000000000000000", ], "chainId": 1, - "fromBlock": undefined, + "fromBlock": 0, "include": [], - "toBlock": undefined, + "toBlock": Infinity, "topic0": null, "topic1": null, "topic2": null, @@ -369,11 +369,11 @@ test("insertIntervals() merges duplicates", async (context) => { Map { { "chainId": 1, - "fromBlock": undefined, + "fromBlock": 0, "include": [], "interval": 1, "offset": 0, - "toBlock": undefined, + "toBlock": Infinity, "type": "block", } => [ { @@ -435,9 +435,9 @@ test("insertIntervals() preserves fragments", async (context) => { "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", ], "chainId": 1, - "fromBlock": undefined, + "fromBlock": 0, "include": [], - "toBlock": undefined, + "toBlock": Infinity, "topic0": null, "topic1": null, "topic2": null,