From 4ef4380ceb65d6255a57b646572ff6f552af2a60 Mon Sep 17 00:00:00 2001 From: benesjan Date: Tue, 17 Dec 2024 18:07:31 +0000 Subject: [PATCH 1/5] refactor: getLogsByTags request optimization in syncTaggedLogsAsSender --- .../pxe/src/simulator_oracle/index.ts | 75 +++++++++---------- 1 file changed, 36 insertions(+), 39 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index acd71477f52..144602f5695 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -353,59 +353,56 @@ export class SimulatorOracle implements DBOracle { recipient: AztecAddress, ): Promise { const appTaggingSecret = await this.#calculateAppTaggingSecret(contractAddress, sender, recipient); - let [currentIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]); - - const INDEX_OFFSET = 10; - - let previousEmptyBack = 0; - let currentEmptyBack = 0; - let currentEmptyFront: number; - - // The below code is trying to find the index of the start of the first window in which for all elements of window, we do not see logs. - // We take our window size, and fetch the node for these logs. We store both the amount of empty consecutive slots from the front and the back. - // We use our current empty consecutive slots from the front, as well as the previous consecutive empty slots from the back to see if we ever hit a time where there - // is a window in which we see the combination of them to be greater than the window's size. If true, we rewind current index to the start of said window and use it. - // Assuming two windows of 5: - // [0, 1, 0, 1, 0], [0, 0, 0, 0, 0] - // We can see that when processing the second window, the previous amount of empty slots from the back of the window (1), added with the empty elements from the front of the window (5) - // is greater than 5 (6) and therefore we have found a window to use. - // We simply need to take the number of elements (10) - the size of the window (5) - the number of consecutive empty elements from the back of the last window (1) = 4; - // This is the first index of our desired window. - // Note that if we ever see a situation like so: - // [0, 1, 0, 1, 0], [0, 0, 0, 0, 1] - // This also returns the correct index (4), but this is indicative of a problem / desync. i.e. we should never have a window that has a log that exists after the window. + const [oldIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]); + // This algorithm works such that: + // 1. If we find minimum consecutive empty logs in a window of logs we set the index to the index of the last log + // we found and quit. + // 2. If we don't find minimum consecutive empty logs in a window of logs we slide the window to latest log index + // and repeat the process. + const WINDOW_SIZE = 20; + const MIN_CONSECUTIVE_EMPTY_LOGS = WINDOW_SIZE / 2; + + let [numConsecutiveEmptyLogs, currentIndex] = [0, oldIndex]; do { - const currentTags = [...new Array(INDEX_OFFSET)].map((_, i) => { + // We compute the tags for the current window of indexes + const currentTags = [...new Array(WINDOW_SIZE)].map((_, i) => { const indexedAppTaggingSecret = new IndexedTaggingSecret(appTaggingSecret, currentIndex + i); return indexedAppTaggingSecret.computeSiloedTag(recipient, contractAddress); }); - previousEmptyBack = currentEmptyBack; + // We fetch the logs for the tags const possibleLogs = await this.aztecNode.getLogsByTags(currentTags); - const indexOfFirstLog = possibleLogs.findIndex(possibleLog => possibleLog.length !== 0); - currentEmptyFront = indexOfFirstLog === -1 ? INDEX_OFFSET : indexOfFirstLog; - + // We find the index of the last log in the window that is not empty const indexOfLastLog = possibleLogs.findLastIndex(possibleLog => possibleLog.length !== 0); - currentEmptyBack = indexOfLastLog === -1 ? INDEX_OFFSET : INDEX_OFFSET - 1 - indexOfLastLog; - currentIndex += INDEX_OFFSET; - } while (currentEmptyFront + previousEmptyBack < INDEX_OFFSET); + if (indexOfLastLog !== -1) { + // We didn't find any logs in the current window so we stop looking + break; + } - // We unwind the entire current window and the amount of consecutive empty slots from the previous window - const newIndex = currentIndex - (INDEX_OFFSET + previousEmptyBack); + // We move the current index to that of the last log we found + currentIndex += indexOfLastLog + 1; - await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, newIndex)]); + // We compute the number of consecutive empty logs we found and repeat the process if we haven't found enough. + numConsecutiveEmptyLogs = WINDOW_SIZE - indexOfLastLog - 1; + } while (numConsecutiveEmptyLogs >= MIN_CONSECUTIVE_EMPTY_LOGS); const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); - this.log.debug(`Syncing logs for sender ${sender} at contract ${contractName}(${contractAddress})`, { - sender, - secret: appTaggingSecret, - index: currentIndex, - contractName, - contractAddress, - }); + if (currentIndex !== oldIndex) { + await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, currentIndex)]); + + this.log.debug(`Syncing logs for sender ${sender} at contract ${contractName}(${contractAddress})`, { + sender, + secret: appTaggingSecret, + index: currentIndex, + contractName, + contractAddress, + }); + } else { + this.log.debug(`No new logs found for sender ${sender} at contract ${contractName}(${contractAddress})`); + } } /** From 1bdac9e0540ffb195bb5c2658e9f85f2ee35ce0f Mon Sep 17 00:00:00 2001 From: benesjan Date: Tue, 17 Dec 2024 18:39:06 +0000 Subject: [PATCH 2/5] WIP --- yarn-project/pxe/src/simulator_oracle/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 144602f5695..614ce0be06e 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -360,8 +360,8 @@ export class SimulatorOracle implements DBOracle { // we found and quit. // 2. If we don't find minimum consecutive empty logs in a window of logs we slide the window to latest log index // and repeat the process. - const WINDOW_SIZE = 20; - const MIN_CONSECUTIVE_EMPTY_LOGS = WINDOW_SIZE / 2; + const MIN_CONSECUTIVE_EMPTY_LOGS = 10; + const WINDOW_SIZE = MIN_CONSECUTIVE_EMPTY_LOGS * 2; let [numConsecutiveEmptyLogs, currentIndex] = [0, oldIndex]; do { From b497ba0ef6ea92296334560bf0afad6bc32037b8 Mon Sep 17 00:00:00 2001 From: benesjan Date: Tue, 17 Dec 2024 18:44:42 +0000 Subject: [PATCH 3/5] fix --- yarn-project/pxe/src/simulator_oracle/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 614ce0be06e..18463edca44 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -377,8 +377,8 @@ export class SimulatorOracle implements DBOracle { // We find the index of the last log in the window that is not empty const indexOfLastLog = possibleLogs.findLastIndex(possibleLog => possibleLog.length !== 0); - if (indexOfLastLog !== -1) { - // We didn't find any logs in the current window so we stop looking + if (indexOfLastLog === -1) { + // We haven't found any logs in the current window so we stop looking break; } From 6dec4f5eb279762b3f62209642998ba2e720d4b7 Mon Sep 17 00:00:00 2001 From: benesjan Date: Tue, 17 Dec 2024 19:08:38 +0000 Subject: [PATCH 4/5] fix --- yarn-project/pxe/src/simulator_oracle/index.ts | 2 +- .../pxe/src/simulator_oracle/simulator_oracle.test.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 18463edca44..470324d96de 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -387,7 +387,7 @@ export class SimulatorOracle implements DBOracle { // We compute the number of consecutive empty logs we found and repeat the process if we haven't found enough. numConsecutiveEmptyLogs = WINDOW_SIZE - indexOfLastLog - 1; - } while (numConsecutiveEmptyLogs >= MIN_CONSECUTIVE_EMPTY_LOGS); + } while (numConsecutiveEmptyLogs < MIN_CONSECUTIVE_EMPTY_LOGS); const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); if (currentIndex !== oldIndex) { diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index 722eaf8e944..8f4f94a01f4 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -292,12 +292,13 @@ describe('Simulator oracle', () => { let indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); - // Two windows are fetch for each sender - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2); + // Only 1 window is obtained for each sender + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS); aztecNode.getLogsByTags.mockReset(); - // We add more logs at the end of the window to make sure we only detect them and bump the indexes if it lies within our window - senderOffset = 10; + // We add more logs to the second half of the window to test that a second iteration in `syncTaggedLogsAsSender` + // is handled correctly. + senderOffset = 11; generateMockLogs(senderOffset); for (let i = 0; i < senders.length; i++) { await simulatorOracle.syncTaggedLogsAsSender( @@ -308,7 +309,7 @@ describe('Simulator oracle', () => { } indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); - expect(indexesAsSenderAfterSync).toStrictEqual([11, 11, 11, 11, 11, 12, 12, 12, 12, 12]); + expect(indexesAsSenderAfterSync).toStrictEqual([12, 12, 12, 12, 12, 13, 13, 13, 13, 13]); expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2); }); From e19a1058949071423c3baf4189acdd09aec6d868 Mon Sep 17 00:00:00 2001 From: benesjan Date: Tue, 17 Dec 2024 19:31:13 +0000 Subject: [PATCH 5/5] naming fixes --- .../pxe/src/simulator_oracle/index.ts | 5 +- .../simulator_oracle/simulator_oracle.test.ts | 60 ++++++++++--------- .../pxe/src/simulator_oracle/tagging_utils.ts | 3 + 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 470324d96de..bb3885695a4 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -38,7 +38,7 @@ import { type IncomingNoteDao } from '../database/incoming_note_dao.js'; import { type PxeDatabase } from '../database/index.js'; import { produceNoteDaos } from '../note_decryption_utils/produce_note_daos.js'; import { getAcirSimulator } from '../simulator/index.js'; -import { getIndexedTaggingSecretsForTheWindow, getInitialIndexesMap } from './tagging_utils.js'; +import { WINDOW_HALF_SIZE, getIndexedTaggingSecretsForTheWindow, getInitialIndexesMap } from './tagging_utils.js'; /** * A data oracle that provides information needed for simulating a transaction. @@ -418,9 +418,6 @@ export class SimulatorOracle implements DBOracle { maxBlockNumber: number, scopes?: AztecAddress[], ): Promise> { - // Half the size of the window we slide over the tagging secret indexes. - const WINDOW_HALF_SIZE = 10; - // Ideally this algorithm would be implemented in noir, exposing its building blocks as oracles. // However it is impossible at the moment due to the language not supporting nested slices. // This nesting is necessary because for a given set of tags we don't diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index 8f4f94a01f4..0db352f96a6 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -37,6 +37,7 @@ import { type PxeDatabase } from '../database/index.js'; import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { ContractDataOracle } from '../index.js'; import { SimulatorOracle } from './index.js'; +import { WINDOW_HALF_SIZE } from './tagging_utils.js'; const TXS_PER_BLOCK = 4; const NUM_NOTE_HASHES_PER_BLOCK = TXS_PER_BLOCK * MAX_NOTE_HASHES_PER_TX; @@ -138,16 +139,15 @@ describe('Simulator oracle', () => { describe('sync tagged logs', () => { const NUM_SENDERS = 10; - const SENDER_OFFSET_WINDOW_SIZE = 10; let senders: { completeAddress: CompleteAddress; ivsk: Fq; secretKey: Fr }[]; - function generateMockLogs(senderOffset: number) { + function generateMockLogs(tagIndex: number) { const logs: { [k: string]: TxScopedL2Log[] } = {}; - // Add a random note from every address in the address book for our account with index senderOffset + // Add a random note from every address in the address book for our account with index tagIndex // Compute the tag as sender (knowledge of preaddress and ivsk) for (const sender of senders) { - const tag = computeSiloedTagForIndex(sender, recipient.address, contractAddress, senderOffset); + const tag = computeSiloedTagForIndex(sender, recipient.address, contractAddress, tagIndex); const blockNumber = 1; const randomNote = new MockNoteRequest( getRandomNoteLogPayload(tag, contractAddress), @@ -164,18 +164,18 @@ describe('Simulator oracle', () => { // Add a random note from the first sender in the address book, repeating the tag // Compute the tag as sender (knowledge of preaddress and ivsk) const firstSender = senders[0]; - const tag = computeSiloedTagForIndex(firstSender, recipient.address, contractAddress, senderOffset); + const tag = computeSiloedTagForIndex(firstSender, recipient.address, contractAddress, tagIndex); const payload = getRandomNoteLogPayload(tag, contractAddress); const logData = payload.generatePayload(GrumpkinScalar.random(), recipient.address).toBuffer(); const log = new TxScopedL2Log(TxHash.random(), 1, 0, false, logData); logs[tag.toString()].push(log); // Accumulated logs intended for recipient: NUM_SENDERS + 1 - // Add a random note from half the address book for our account with index senderOffset + 1 + // Add a random note from half the address book for our account with index tagIndex + 1 // Compute the tag as sender (knowledge of preaddress and ivsk) for (let i = NUM_SENDERS / 2; i < NUM_SENDERS; i++) { const sender = senders[i]; - const tag = computeSiloedTagForIndex(sender, recipient.address, contractAddress, senderOffset + 1); + const tag = computeSiloedTagForIndex(sender, recipient.address, contractAddress, tagIndex + 1); const blockNumber = 2; const randomNote = new MockNoteRequest( getRandomNoteLogPayload(tag, contractAddress), @@ -189,13 +189,13 @@ describe('Simulator oracle', () => { } // Accumulated logs intended for recipient: NUM_SENDERS + 1 + NUM_SENDERS / 2 - // Add a random note from every address in the address book for a random recipient with index senderOffset + // Add a random note from every address in the address book for a random recipient with index tagIndex // Compute the tag as sender (knowledge of preaddress and ivsk) for (const sender of senders) { const keys = deriveKeys(Fr.random()); const partialAddress = Fr.random(); const randomRecipient = computeAddress(keys.publicKeys, partialAddress); - const tag = computeSiloedTagForIndex(sender, randomRecipient, contractAddress, senderOffset); + const tag = computeSiloedTagForIndex(sender, randomRecipient, contractAddress, tagIndex); const blockNumber = 3; const randomNote = new MockNoteRequest( getRandomNoteLogPayload(tag, contractAddress), @@ -232,8 +232,8 @@ describe('Simulator oracle', () => { }); it('should sync tagged logs', async () => { - const senderOffset = 0; - generateMockLogs(senderOffset); + const tagIndex = 0; + generateMockLogs(tagIndex); const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); // We expect to have all logs intended for the recipient, one per sender + 1 with a duplicated tag for the first // one + half of the logs for the second index @@ -266,8 +266,8 @@ describe('Simulator oracle', () => { await keyStore.addAccount(sender.secretKey, sender.completeAddress.partialAddress); } - let senderOffset = 0; - generateMockLogs(senderOffset); + let tagIndex = 0; + generateMockLogs(tagIndex); // Recompute the secrets (as recipient) to ensure indexes are updated const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); @@ -298,8 +298,8 @@ describe('Simulator oracle', () => { // We add more logs to the second half of the window to test that a second iteration in `syncTaggedLogsAsSender` // is handled correctly. - senderOffset = 11; - generateMockLogs(senderOffset); + tagIndex = 11; + generateMockLogs(tagIndex); for (let i = 0; i < senders.length; i++) { await simulatorOracle.syncTaggedLogsAsSender( contractAddress, @@ -315,8 +315,8 @@ describe('Simulator oracle', () => { }); it('should sync tagged logs with a sender index offset', async () => { - const senderOffset = 5; - generateMockLogs(senderOffset); + const tagIndex = 5; + generateMockLogs(tagIndex); const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); // We expect to have all logs intended for the recipient, one per sender + 1 with a duplicated tag for the first one + half of the logs for the second index expect(syncedLogs.get(recipient.address.toString())).toHaveLength(NUM_SENDERS + 1 + NUM_SENDERS / 2); @@ -342,8 +342,8 @@ describe('Simulator oracle', () => { }); it("should sync tagged logs for which indexes are not updated if they're inside the window", async () => { - const senderOffset = 1; - generateMockLogs(senderOffset); + const tagIndex = 1; + generateMockLogs(tagIndex); // Recompute the secrets (as recipient) to update indexes const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); @@ -362,8 +362,8 @@ describe('Simulator oracle', () => { expect(syncedLogs.get(recipient.address.toString())).toHaveLength(NUM_SENDERS + 1 + NUM_SENDERS / 2); // First sender should have 2 logs, but keep index 2 since they were built using the same tag - // Next 4 senders should also have index 2 = offset + 1 - // Last 5 senders should have index 3 = offset + 2 + // Next 4 senders should also have index 2 = tagIndex + 1 + // Last 5 senders should have index 3 = tagIndex + 2 const indexes = await database.getTaggingSecretsIndexesAsRecipient(secrets); expect(indexes).toHaveLength(NUM_SENDERS); @@ -375,8 +375,8 @@ describe('Simulator oracle', () => { }); it("should not sync tagged logs for which indexes are not updated if they're outside the window", async () => { - const senderOffset = 0; - generateMockLogs(senderOffset); + const tagIndex = 0; + generateMockLogs(tagIndex); // Recompute the secrets (as recipient) to update indexes const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); @@ -385,8 +385,10 @@ describe('Simulator oracle', () => { return poseidon2Hash([firstSenderSecretPoint.x, firstSenderSecretPoint.y, contractAddress]); }); + // We set the indexes to WINDOW_HALF_SIZE + 1 so that it's outside the window and for this reason no updates + // should be triggered. await database.setTaggingSecretsIndexesAsRecipient( - secrets.map(secret => new IndexedTaggingSecret(secret, SENDER_OFFSET_WINDOW_SIZE + 1)), + secrets.map(secret => new IndexedTaggingSecret(secret, WINDOW_HALF_SIZE + 1)), ); const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); @@ -405,8 +407,8 @@ describe('Simulator oracle', () => { }); it('should sync tagged logs from scratch after a DB wipe', async () => { - const senderOffset = 0; - generateMockLogs(senderOffset); + const tagIndex = 0; + generateMockLogs(tagIndex); // Recompute the secrets (as recipient) to update indexes const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); @@ -416,7 +418,7 @@ describe('Simulator oracle', () => { }); await database.setTaggingSecretsIndexesAsRecipient( - secrets.map(secret => new IndexedTaggingSecret(secret, SENDER_OFFSET_WINDOW_SIZE + 2)), + secrets.map(secret => new IndexedTaggingSecret(secret, WINDOW_HALF_SIZE + 2)), ); let syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); @@ -448,8 +450,8 @@ describe('Simulator oracle', () => { }); it('should not sync tagged logs with a blockNumber > maxBlockNumber', async () => { - const senderOffset = 0; - generateMockLogs(senderOffset); + const tagIndex = 0; + generateMockLogs(tagIndex); const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 1); // Only NUM_SENDERS + 1 logs should be synched, since the rest have blockNumber > 1 diff --git a/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts b/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts index 82759eaf038..6cf121bcd7b 100644 --- a/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts +++ b/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts @@ -1,5 +1,8 @@ import { type Fr, IndexedTaggingSecret } from '@aztec/circuits.js'; +// Half the size of the window we slide over the tagging secret indexes. +export const WINDOW_HALF_SIZE = 10; + export function getIndexedTaggingSecretsForTheWindow( secretsAndWindows: { appTaggingSecret: Fr; leftMostIndex: number; rightMostIndex: number }[], ): IndexedTaggingSecret[] {