Skip to content

Commit

Permalink
chore: Batch archiver requests
Browse files Browse the repository at this point in the history 
Instead of downloading every block and keeping it in memory, download in
batches of (configurable) 100 blocks and store them progressively.
  • Loading branch information
@spalladino
spalladino committed Dec 6, 2024
1 parent 4fcbc59 commit 170ba2f
Show file tree
Hide file tree
Showing 7 changed files with 220 additions and 299 deletions.
2 changes: 0 additions & 2 deletions yarn-project/archiver/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@
"formatting": "run -T prettier --check ./src && run -T eslint ./src",
"formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
"test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests",
"start": "node ./dest",
"start:dev": "tsc-watch -p tsconfig.json --onSuccess 'yarn start'",
"test:integration": "concurrently -k -s first -c reset,dim -n test,anvil \"yarn test:integration:run\" \"anvil\"",
"test:integration:run": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --config jest.integration.config.json"
},
Expand Down
205 changes: 91 additions & 114 deletions yarn-project/archiver/src/archiver/archiver.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,27 +56,50 @@ describe('Archiver', () => {
let archiver: Archiver;
let blocks: L2Block[];

let l2BlockProposedLogs: Log<bigint, number, false, undefined, true, typeof RollupAbi, 'L2BlockProposed'>[];
let l2MessageSentLogs: Log<bigint, number, false, undefined, true, typeof InboxAbi, 'MessageSent'>[];

const GENESIS_ROOT = new Fr(GENESIS_ARCHIVE_ROOT).toString();

beforeEach(() => {
now = +new Date();
publicClient = mock<PublicClient<HttpTransport, Chain>>({
// Return a block with a reasonable timestamp
getBlock: ((args: any) => ({
timestamp: args.blockNumber * BigInt(DefaultL1ContractsConfig.ethereumSlotDuration) + BigInt(now),
})) as any,
// Return the logs mocked whenever the public client is queried
getLogs: ((args: any) => {
let logs = undefined;
if (args!.event!.name === 'MessageSent') {
logs = l2MessageSentLogs;
} else if (args!.event!.name === 'L2BlockProposed') {
logs = l2BlockProposedLogs;
} else {
throw new Error(`Unknown event: ${args!.event!.name}`);
}
return Promise.resolve(
logs.filter(log => log.blockNumber >= args.fromBlock && log.blockNumber <= args.toBlock),
);
}) as any,
});

instrumentation = mock({ isEnabled: () => true });
archiverStore = new MemoryArchiverStore(1000);

archiver = new Archiver(
publicClient,
rollupAddress,
inboxAddress,
registryAddress,
{ rollupAddress, inboxAddress, registryAddress },
archiverStore,
1000,
{ pollingIntervalMs: 1000, batchSize: 1000 },
instrumentation,
{
l1GenesisTime: BigInt(now),
l1StartBlock: 0n,
epochDuration: 4,
slotDuration: 24,
ethereumSlotDuration: 12,
},
);

blocks = blockNumbers.map(x => L2Block.random(x, txsPerBlock, x + 1, 2));
Expand All @@ -97,6 +120,9 @@ describe('Archiver', () => {

inboxRead = mock<MockInboxContractRead>();
((archiver as any).inbox as any).read = inboxRead;

l2MessageSentLogs = [];
l2BlockProposedLogs = [];
});

afterEach(async () => {
Expand Down Expand Up @@ -127,27 +153,16 @@ describe('Archiver', () => {

inboxRead.totalMessagesInserted.mockResolvedValueOnce(2n).mockResolvedValueOnce(6n);

mockGetLogs({
messageSent: [
makeMessageSentEventWithIndexInL2BlockSubtree(98n, 1n, 0n),
makeMessageSentEventWithIndexInL2BlockSubtree(99n, 1n, 1n),
],
L2BlockProposed: [makeL2BlockProposedEvent(101n, 1n, blocks[0].archive.root.toString())],
});

mockGetLogs({
messageSent: [
makeMessageSentEventWithIndexInL2BlockSubtree(2504n, 2n, 0n),
makeMessageSentEventWithIndexInL2BlockSubtree(2505n, 2n, 1n),
makeMessageSentEventWithIndexInL2BlockSubtree(2505n, 2n, 2n),
makeMessageSentEventWithIndexInL2BlockSubtree(2506n, 3n, 1n),
],
L2BlockProposed: [
makeL2BlockProposedEvent(2510n, 2n, blocks[1].archive.root.toString()),
makeL2BlockProposedEvent(2520n, 3n, blocks[2].archive.root.toString()),
],
});
makeMessageSentEvent(98n, 1n, 0n);
makeMessageSentEvent(99n, 1n, 1n);
makeL2BlockProposedEvent(101n, 1n, blocks[0].archive.root.toString());

makeMessageSentEvent(2504n, 2n, 0n);
makeMessageSentEvent(2505n, 2n, 1n);
makeMessageSentEvent(2505n, 2n, 2n);
makeMessageSentEvent(2506n, 3n, 1n);
makeL2BlockProposedEvent(2510n, 2n, blocks[1].archive.root.toString());
makeL2BlockProposedEvent(2520n, 3n, blocks[2].archive.root.toString());
publicClient.getTransaction.mockResolvedValueOnce(rollupTxs[0]);

rollupTxs.slice(1).forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx));
Expand Down Expand Up @@ -228,17 +243,11 @@ describe('Archiver', () => {

inboxRead.totalMessagesInserted.mockResolvedValueOnce(2n).mockResolvedValueOnce(2n);

mockGetLogs({
messageSent: [
makeMessageSentEventWithIndexInL2BlockSubtree(66n, 1n, 0n),
makeMessageSentEventWithIndexInL2BlockSubtree(68n, 1n, 1n),
],
L2BlockProposed: [
makeL2BlockProposedEvent(70n, 1n, blocks[0].archive.root.toString()),
makeL2BlockProposedEvent(80n, 2n, blocks[1].archive.root.toString()),
makeL2BlockProposedEvent(90n, 3n, badArchive),
],
});
makeMessageSentEvent(66n, 1n, 0n);
makeMessageSentEvent(68n, 1n, 1n);
makeL2BlockProposedEvent(70n, 1n, blocks[0].archive.root.toString());
makeL2BlockProposedEvent(80n, 2n, blocks[1].archive.root.toString());
makeL2BlockProposedEvent(90n, 3n, badArchive);

rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx));

Expand All @@ -250,8 +259,10 @@ describe('Archiver', () => {

latestBlockNum = await archiver.getBlockNumber();
expect(latestBlockNum).toEqual(numL2BlocksInTest);
const errorMessage = `Archive mismatch matching, ignoring block ${3} with archive: ${badArchive}, expected ${blocks[2].archive.root.toString()}`;
expect(loggerSpy).toHaveBeenCalledWith(errorMessage);
expect(loggerSpy).toHaveBeenCalledWith(expect.stringMatching(/archive root mismatch/i), {
actual: badArchive,
expected: blocks[2].archive.root.toString(),
});
}, 10_000);

it('skip event search if no changes found', async () => {
Expand All @@ -271,16 +282,10 @@ describe('Archiver', () => {

inboxRead.totalMessagesInserted.mockResolvedValueOnce(0n).mockResolvedValueOnce(2n);

mockGetLogs({
messageSent: [
makeMessageSentEventWithIndexInL2BlockSubtree(66n, 1n, 0n),
makeMessageSentEventWithIndexInL2BlockSubtree(68n, 1n, 1n),
],
L2BlockProposed: [
makeL2BlockProposedEvent(70n, 1n, blocks[0].archive.root.toString()),
makeL2BlockProposedEvent(80n, 2n, blocks[1].archive.root.toString()),
],
});
makeMessageSentEvent(66n, 1n, 0n);
makeMessageSentEvent(68n, 1n, 1n);
makeL2BlockProposedEvent(70n, 1n, blocks[0].archive.root.toString());
makeL2BlockProposedEvent(80n, 2n, blocks[1].archive.root.toString());

rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx));

Expand All @@ -292,10 +297,7 @@ describe('Archiver', () => {

latestBlockNum = await archiver.getBlockNumber();
expect(latestBlockNum).toEqual(numL2BlocksInTest);

// For some reason, this is 1-indexed.
expect(loggerSpy).toHaveBeenNthCalledWith(1, `Retrieved no new L1 to L2 messages between L1 blocks 1 and 50.`);
expect(loggerSpy).toHaveBeenNthCalledWith(2, `No blocks to retrieve from 1 to 50`);
expect(loggerSpy).toHaveBeenCalledWith(`No blocks to retrieve from 1 to 50`);
}, 10_000);

it('handles L2 reorg', async () => {
Expand Down Expand Up @@ -328,16 +330,10 @@ describe('Archiver', () => {
.mockResolvedValueOnce(2n)
.mockResolvedValueOnce(2n);

mockGetLogs({
messageSent: [
makeMessageSentEventWithIndexInL2BlockSubtree(66n, 1n, 0n),
makeMessageSentEventWithIndexInL2BlockSubtree(68n, 1n, 1n),
],
L2BlockProposed: [
makeL2BlockProposedEvent(70n, 1n, blocks[0].archive.root.toString()),
makeL2BlockProposedEvent(80n, 2n, blocks[1].archive.root.toString()),
],
});
makeMessageSentEvent(66n, 1n, 0n);
makeMessageSentEvent(68n, 1n, 1n);
makeL2BlockProposedEvent(70n, 1n, blocks[0].archive.root.toString());
makeL2BlockProposedEvent(80n, 2n, blocks[1].archive.root.toString());

rollupTxs.forEach(tx => publicClient.getTransaction.mockResolvedValueOnce(tx));

Expand All @@ -350,14 +346,12 @@ describe('Archiver', () => {
latestBlockNum = await archiver.getBlockNumber();
expect(latestBlockNum).toEqual(numL2BlocksInTest);

// For some reason, this is 1-indexed.
expect(loggerSpy).toHaveBeenNthCalledWith(1, `Retrieved no new L1 to L2 messages between L1 blocks 1 and 50.`);
expect(loggerSpy).toHaveBeenNthCalledWith(2, `No blocks to retrieve from 1 to 50`);
expect(loggerSpy).toHaveBeenCalledWith(`No blocks to retrieve from 1 to 50`);

// Lets take a look to see if we can find re-org stuff!
await sleep(1000);

expect(loggerSpy).toHaveBeenNthCalledWith(9, `L2 prune has been detected.`);
expect(loggerSpy).toHaveBeenCalledWith(`L2 prune has been detected.`);

// Should also see the block number be reduced
latestBlockNum = await archiver.getBlockNumber();
Expand All @@ -375,57 +369,40 @@ describe('Archiver', () => {
// TODO(palla/reorg): Add a unit test for the archiver handleEpochPrune
xit('handles an upcoming L2 prune', () => {});

// logs should be created in order of how archiver syncs.
const mockGetLogs = (logs: {
messageSent?: ReturnType<typeof makeMessageSentEventWithIndexInL2BlockSubtree>[];
L2BlockProposed?: ReturnType<typeof makeL2BlockProposedEvent>[];
}) => {
if (logs.messageSent) {
publicClient.getLogs.mockResolvedValueOnce(logs.messageSent);
}
if (logs.L2BlockProposed) {
publicClient.getLogs.mockResolvedValueOnce(logs.L2BlockProposed);
}
/**
* Makes a fake L2BlockProposed event for testing purposes and registers it to be returned by the public client.
* @param l1BlockNum - L1 block number.
* @param l2BlockNum - L2 Block number.
*/
const makeL2BlockProposedEvent = (l1BlockNum: bigint, l2BlockNum: bigint, archive: `0x${string}`) => {
const log = {
blockNumber: l1BlockNum,
args: { blockNumber: l2BlockNum, archive },
transactionHash: `0x${l2BlockNum}`,
} as Log<bigint, number, false, undefined, true, typeof RollupAbi, 'L2BlockProposed'>;
l2BlockProposedLogs.push(log);
};
});

/**
* Makes a fake L2BlockProposed event for testing purposes.
* @param l1BlockNum - L1 block number.
* @param l2BlockNum - L2 Block number.
* @returns An L2BlockProposed event log.
*/
function makeL2BlockProposedEvent(l1BlockNum: bigint, l2BlockNum: bigint, archive: `0x${string}`) {
return {
blockNumber: l1BlockNum,
args: { blockNumber: l2BlockNum, archive },
transactionHash: `0x${l2BlockNum}`,
} as Log<bigint, number, false, undefined, true, typeof RollupAbi, 'L2BlockProposed'>;
}

/**
* Makes fake L1ToL2 MessageSent events for testing purposes.
* @param l1BlockNum - L1 block number.
* @param l2BlockNumber - The L2 block number for which the message was included.
* @param indexInSubtree - the index in the l2Block's subtree in the L1 to L2 Messages Tree.
* @returns MessageSent event logs.
*/
function makeMessageSentEventWithIndexInL2BlockSubtree(
l1BlockNum: bigint,
l2BlockNumber: bigint,
indexInSubtree: bigint,
) {
const index = indexInSubtree + InboxLeaf.smallestIndexFromL2Block(l2BlockNumber);
return {
blockNumber: l1BlockNum,
args: {
l2BlockNumber,
index,
hash: Fr.random().toString(),
},
transactionHash: `0x${l1BlockNum}`,
} as Log<bigint, number, false, undefined, true, typeof InboxAbi, 'MessageSent'>;
}
/**
* Makes fake L1ToL2 MessageSent events for testing purposes and registers it to be returned by the public client.
* @param l1BlockNum - L1 block number.
* @param l2BlockNumber - The L2 block number for which the message was included.
* @param indexInSubtree - the index in the l2Block's subtree in the L1 to L2 Messages Tree.
*/
const makeMessageSentEvent = (l1BlockNum: bigint, l2BlockNumber: bigint, indexInSubtree: bigint) => {
const index = indexInSubtree + InboxLeaf.smallestIndexFromL2Block(l2BlockNumber);
const log = {
blockNumber: l1BlockNum,
args: {
l2BlockNumber,
index,
hash: Fr.random().toString(),
},
transactionHash: `0x${l1BlockNum}`,
} as Log<bigint, number, false, undefined, true, typeof InboxAbi, 'MessageSent'>;
l2MessageSentLogs.push(log);
};
});

/**
* Makes a fake rollup tx for testing purposes.
Expand Down
Loading

0 comments on commit 170ba2f

Please sign in to comment.