From 0d1d233aaf757b9c60fc376072fed659fe5e6231 Mon Sep 17 00:00:00 2001 From: sklppy88 Date: Wed, 20 Nov 2024 14:52:38 +0000 Subject: [PATCH 1/5] init --- .../pxe/src/database/kv_pxe_database.ts | 27 +++--- yarn-project/pxe/src/database/pxe_database.ts | 7 +- .../pxe/src/simulator_oracle/index.ts | 84 ++++++++++++++++--- yarn-project/txe/src/oracle/txe_oracle.ts | 5 +- 4 files changed, 86 insertions(+), 37 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 96a27865194..58d4ab5d460 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -668,26 +668,21 @@ export class KVPxeDatabase implements PxeDatabase { return incomingNotesSize + outgoingNotesSize + treeRootsSize + authWitsSize + addressesSize; } - async incrementTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise { - await this.#incrementTaggingSecretsIndexes(appTaggingSecrets, this.#taggingSecretIndexesForSenders); + async setTaggingSecretsIndexesAsSender(indexedSecrets: IndexedTaggingSecret[]): Promise { + await this.#setTaggingSecretsIndexes(indexedSecrets, this.#taggingSecretIndexesForSenders); } - async #incrementTaggingSecretsIndexes(appTaggingSecrets: Fr[], storageMap: AztecMap): Promise { - const indexes = await this.#getTaggingSecretsIndexes(appTaggingSecrets, storageMap); - await this.db.transaction(() => { - indexes.forEach((taggingSecretIndex, listIndex) => { - const nextIndex = taggingSecretIndex + 1; - void storageMap.set(appTaggingSecrets[listIndex].toString(), nextIndex); - }); - }); + async setTaggingSecretsIndexesAsRecipient(indexedSecrets: IndexedTaggingSecret[]): Promise { + await this.#setTaggingSecretsIndexes(indexedSecrets, this.#taggingSecretIndexesForRecipients); } - async setTaggingSecretsIndexesAsRecipient(indexedSecrets: IndexedTaggingSecret[]): Promise { - await this.db.transaction(() => { - indexedSecrets.forEach(indexedSecret => { - void this.#taggingSecretIndexesForRecipients.set(indexedSecret.secret.toString(), indexedSecret.index); - }); - }); + #setTaggingSecretsIndexes( + indexedSecrets: IndexedTaggingSecret[], + storageMap: AztecMap, + ): Promise[]> { + return this.db.transaction(() => + indexedSecrets.map(indexedSecret => storageMap.set(indexedSecret.secret.toString(), indexedSecret.index)), + ); } async getTaggingSecretsIndexesAsRecipient(appTaggingSecrets: Fr[]) { diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index 1e7293c33ae..e46239aba47 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -201,12 +201,7 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD */ getTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise; - /** - * Increments the index for the provided app siloed tagging secrets in the senders database - * To be used when the generated tags have been used as sender - * @param appTaggingSecrets - The app siloed tagging secrets. - */ - incrementTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise; + setTaggingSecretsIndexesAsSender(indexedTaggingSecrets: IndexedTaggingSecret[]): Promise; /** * Sets the index for the provided app siloed tagging secrets diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 72956f09ce4..af183e556a0 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -268,8 +268,11 @@ export class SimulatorOracle implements DBOracle { sender: AztecAddress, recipient: AztecAddress, ): Promise { + await this.syncTaggedLogsAsSender(contractAddress, sender, recipient); + const secret = await this.#calculateTaggingSecret(contractAddress, sender, recipient); const [index] = await this.db.getTaggingSecretsIndexesAsSender([secret]); + return new IndexedTaggingSecret(secret, index); } @@ -289,7 +292,9 @@ export class SimulatorOracle implements DBOracle { this.log.verbose( `Incrementing secret ${secret} as sender ${sender} for recipient: ${recipient} at contract: ${contractName}(${contractAddress})`, ); - await this.db.incrementTaggingSecretsIndexesAsSender([secret]); + + const [index] = await this.db.getTaggingSecretsIndexesAsSender([secret]); + await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(secret, index)]); } async #calculateTaggingSecret(contractAddress: AztecAddress, sender: AztecAddress, recipient: AztecAddress) { @@ -329,6 +334,21 @@ export class SimulatorOracle implements DBOracle { return appTaggingSecrets.map((secret, i) => new IndexedTaggingSecret(secret, indexes[i])); } + async #getAppTaggingSecretAsSender( + contractAddress: AztecAddress, + sender: AztecAddress, + recipient: AztecAddress, + ): Promise { + const senderCompleteAddress = await this.getCompleteAddress(sender); + const senderIvsk = await this.keyStore.getMasterIncomingViewingSecretKey(sender); + + const sharedSecret = computeTaggingSecret(senderCompleteAddress, senderIvsk, recipient); + const appTaggingSecret = poseidon2Hash([sharedSecret.x, sharedSecret.y, contractAddress]); + const [index] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]); + + return new IndexedTaggingSecret(appTaggingSecret, index); + } + /** * Synchronizes the logs tagged with scoped addresses and all the senders in the addressbook. * Returns the unsynched logs and updates the indexes of the secrets used to tag them until there are no more logs to sync. @@ -342,6 +362,34 @@ export class SimulatorOracle implements DBOracle { scopes?: AztecAddress[], ): Promise> { const recipients = scopes ? scopes : await this.keyStore.getAccounts(); + + const result = await this.#syncTaggedLogs(contractAddress, recipients); + + for (const [key, value] of result) { + result.set( + key, + value.filter(log => log.blockNumber <= maxBlockNumber), + ); + } + + return result; + } + + public async syncTaggedLogsAsSender( + contractAddress: AztecAddress, + sender: AztecAddress, + recipient: AztecAddress, + ): Promise> { + const result = await this.#syncTaggedLogs(contractAddress, [recipient], sender); + + return result; + } + + async #syncTaggedLogs( + contractAddress: AztecAddress, + recipients: AztecAddress[], + asSender?: AztecAddress, + ): Promise> { const result = new Map(); const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); for (const recipient of recipients) { @@ -353,7 +401,12 @@ export class SimulatorOracle implements DBOracle { // length, since we don't really know the note they correspond to until we decrypt them. // 1. Get all the secrets for the recipient and sender pairs (#9365) - const appTaggingSecrets = await this.#getAppTaggingSecretsForContacts(contractAddress, recipient); + const appTaggingSecrets: IndexedTaggingSecret[] = []; + if (asSender === undefined) { + appTaggingSecrets.push(...(await this.#getAppTaggingSecretsForContacts(contractAddress, recipient))); + } else { + appTaggingSecrets.push(await this.#getAppTaggingSecretAsSender(contractAddress, asSender, recipient)); + } // 1.1 Set up a sliding window with an offset. Chances are the sender might have messed up // and inadvertedly incremented their index without use getting any logs (for example, in case @@ -431,22 +484,27 @@ export class SimulatorOracle implements DBOracle { newTaggingSecrets.push(newTaggingSecret); } }); - await this.db.setTaggingSecretsIndexesAsRecipient( - Object.keys(secretsToIncrement).map( - secret => new IndexedTaggingSecret(Fr.fromString(secret), secretsToIncrement[secret]), - ), - ); + if (asSender === undefined) { + await this.db.setTaggingSecretsIndexesAsRecipient( + Object.keys(secretsToIncrement).map( + secret => new IndexedTaggingSecret(Fr.fromString(secret), secretsToIncrement[secret]), + ), + ); + } else { + await this.db.setTaggingSecretsIndexesAsSender( + Object.keys(secretsToIncrement).map( + secret => new IndexedTaggingSecret(Fr.fromString(secret), secretsToIncrement[secret]), + ), + ); + } + currentTagggingSecrets = newTaggingSecrets; } result.set( recipient.toString(), - // Remove logs with a block number higher than the max block number - // Duplicates are likely to happen due to the sliding window, so we also filter them out - logs.filter( - (log, index, self) => - log.blockNumber <= maxBlockNumber && index === self.findIndex(otherLog => otherLog.equals(log)), - ), + // Duplicates are likely to happen due to the sliding window, so we filter them out + logs.filter((log, index, self) => index === self.findIndex(otherLog => otherLog.equals(log))), ); } return result; diff --git a/yarn-project/txe/src/oracle/txe_oracle.ts b/yarn-project/txe/src/oracle/txe_oracle.ts index eec5749bf8e..de40b91e8f4 100644 --- a/yarn-project/txe/src/oracle/txe_oracle.ts +++ b/yarn-project/txe/src/oracle/txe_oracle.ts @@ -773,8 +773,9 @@ export class TXE implements TypedOracle { } async incrementAppTaggingSecretIndexAsSender(sender: AztecAddress, recipient: AztecAddress): Promise { - const directionalSecret = await this.#calculateTaggingSecret(this.contractAddress, sender, recipient); - await this.txeDatabase.incrementTaggingSecretsIndexesAsSender([directionalSecret]); + const appSecret = await this.#calculateTaggingSecret(this.contractAddress, sender, recipient); + const [index] = await this.txeDatabase.getTaggingSecretsIndexesAsSender([appSecret]); + await this.txeDatabase.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appSecret, index + 1)]); } async getAppTaggingSecretAsSender(sender: AztecAddress, recipient: AztecAddress): Promise { From 1eb28d97ae7316c14fccb8a9a5c66ae8ef63eb3e Mon Sep 17 00:00:00 2001 From: sklppy88 Date: Wed, 20 Nov 2024 22:51:18 +0000 Subject: [PATCH 2/5] better approach --- .../pxe/src/database/kv_pxe_database.ts | 13 ++- yarn-project/pxe/src/database/pxe_database.ts | 5 + .../pxe/src/simulator_oracle/index.ts | 100 +++++++----------- .../simulator_oracle/simulator_oracle.test.ts | 40 ++++++- 4 files changed, 90 insertions(+), 68 deletions(-) diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 58d4ab5d460..287af7b6bbd 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -676,13 +676,12 @@ export class KVPxeDatabase implements PxeDatabase { await this.#setTaggingSecretsIndexes(indexedSecrets, this.#taggingSecretIndexesForRecipients); } - #setTaggingSecretsIndexes( - indexedSecrets: IndexedTaggingSecret[], - storageMap: AztecMap, - ): Promise[]> { - return this.db.transaction(() => - indexedSecrets.map(indexedSecret => storageMap.set(indexedSecret.secret.toString(), indexedSecret.index)), - ); + #setTaggingSecretsIndexes(indexedSecrets: IndexedTaggingSecret[], storageMap: AztecMap) { + return this.db.transaction(() => { + indexedSecrets.forEach( + indexedSecret => void storageMap.set(indexedSecret.secret.toString(), indexedSecret.index), + ); + }); } async getTaggingSecretsIndexesAsRecipient(appTaggingSecrets: Fr[]) { diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index e46239aba47..8b884041bb9 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -201,6 +201,11 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD */ getTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise; + /** + * Sets the index for the provided app siloed tagging secrets + * To be used when the generated tags have been "seen" as a sender + * @param appTaggingSecrets - The app siloed tagging secrets. + */ setTaggingSecretsIndexesAsSender(indexedTaggingSecrets: IndexedTaggingSecret[]): Promise; /** diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index af183e556a0..8d35b369682 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -294,7 +294,7 @@ export class SimulatorOracle implements DBOracle { ); const [index] = await this.db.getTaggingSecretsIndexesAsSender([secret]); - await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(secret, index)]); + await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(secret, index + 1)]); } async #calculateTaggingSecret(contractAddress: AztecAddress, sender: AztecAddress, recipient: AztecAddress) { @@ -334,19 +334,39 @@ export class SimulatorOracle implements DBOracle { return appTaggingSecrets.map((secret, i) => new IndexedTaggingSecret(secret, indexes[i])); } - async #getAppTaggingSecretAsSender( + /** + * Updates the local index of the shared tagging secret of a sender / recipient pair + * if a log with a larger index is found from the node. + * @param contractAddress - The address of the contract that the logs are tagged for + * @param sender - The address of the sender, we must know the sender's ivsk_m. + * @param recipient - The address of the recipient. + */ + public async syncTaggedLogsAsSender( contractAddress: AztecAddress, sender: AztecAddress, recipient: AztecAddress, - ): Promise { - const senderCompleteAddress = await this.getCompleteAddress(sender); - const senderIvsk = await this.keyStore.getMasterIncomingViewingSecretKey(sender); + ): Promise { + const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); + const appTaggingSecret = await this.#calculateTaggingSecret(contractAddress, sender, recipient); + let [currentIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]); - const sharedSecret = computeTaggingSecret(senderCompleteAddress, senderIvsk, recipient); - const appTaggingSecret = poseidon2Hash([sharedSecret.x, sharedSecret.y, contractAddress]); - const [index] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]); + while (true) { + const currentIndexedAppTaggingSecret = new IndexedTaggingSecret(appTaggingSecret, currentIndex); + const currentTag = currentIndexedAppTaggingSecret.computeTag(recipient); + + const [[possibleLog]] = await this.aztecNode.getLogsByTags([currentTag]); - return new IndexedTaggingSecret(appTaggingSecret, index); + if (possibleLog !== undefined) { + currentIndex++; + } else { + this.log.debug( + `Syncing logs for sender ${sender}, secret ${appTaggingSecret}:${currentIndex} at contract: ${contractName}(${contractAddress})`, + ); + + await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, currentIndex)]); + break; + } + } } /** @@ -362,34 +382,6 @@ export class SimulatorOracle implements DBOracle { scopes?: AztecAddress[], ): Promise> { const recipients = scopes ? scopes : await this.keyStore.getAccounts(); - - const result = await this.#syncTaggedLogs(contractAddress, recipients); - - for (const [key, value] of result) { - result.set( - key, - value.filter(log => log.blockNumber <= maxBlockNumber), - ); - } - - return result; - } - - public async syncTaggedLogsAsSender( - contractAddress: AztecAddress, - sender: AztecAddress, - recipient: AztecAddress, - ): Promise> { - const result = await this.#syncTaggedLogs(contractAddress, [recipient], sender); - - return result; - } - - async #syncTaggedLogs( - contractAddress: AztecAddress, - recipients: AztecAddress[], - asSender?: AztecAddress, - ): Promise> { const result = new Map(); const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); for (const recipient of recipients) { @@ -401,12 +393,7 @@ export class SimulatorOracle implements DBOracle { // length, since we don't really know the note they correspond to until we decrypt them. // 1. Get all the secrets for the recipient and sender pairs (#9365) - const appTaggingSecrets: IndexedTaggingSecret[] = []; - if (asSender === undefined) { - appTaggingSecrets.push(...(await this.#getAppTaggingSecretsForContacts(contractAddress, recipient))); - } else { - appTaggingSecrets.push(await this.#getAppTaggingSecretAsSender(contractAddress, asSender, recipient)); - } + const appTaggingSecrets = await this.#getAppTaggingSecretsForContacts(contractAddress, recipient); // 1.1 Set up a sliding window with an offset. Chances are the sender might have messed up // and inadvertedly incremented their index without use getting any logs (for example, in case @@ -484,27 +471,22 @@ export class SimulatorOracle implements DBOracle { newTaggingSecrets.push(newTaggingSecret); } }); - if (asSender === undefined) { - await this.db.setTaggingSecretsIndexesAsRecipient( - Object.keys(secretsToIncrement).map( - secret => new IndexedTaggingSecret(Fr.fromString(secret), secretsToIncrement[secret]), - ), - ); - } else { - await this.db.setTaggingSecretsIndexesAsSender( - Object.keys(secretsToIncrement).map( - secret => new IndexedTaggingSecret(Fr.fromString(secret), secretsToIncrement[secret]), - ), - ); - } - + await this.db.setTaggingSecretsIndexesAsRecipient( + Object.keys(secretsToIncrement).map( + secret => new IndexedTaggingSecret(Fr.fromString(secret), secretsToIncrement[secret]), + ), + ); currentTagggingSecrets = newTaggingSecrets; } result.set( recipient.toString(), - // Duplicates are likely to happen due to the sliding window, so we filter them out - logs.filter((log, index, self) => index === self.findIndex(otherLog => otherLog.equals(log))), + // Remove logs with a block number higher than the max block number + // Duplicates are likely to happen due to the sliding window, so we also filter them out + logs.filter( + (log, index, self) => + log.blockNumber <= maxBlockNumber && index === self.findIndex(otherLog => otherLog.equals(log)), + ), ); } return result; diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index 6f7a1204e94..002fd32b7f4 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -143,7 +143,7 @@ describe('Simulator oracle', () => { describe('sync tagged logs', () => { const NUM_SENDERS = 10; const SENDER_OFFSET_WINDOW_SIZE = 10; - let senders: { completeAddress: CompleteAddress; ivsk: Fq }[]; + let senders: { completeAddress: CompleteAddress; ivsk: Fq; secretKey: Fr }[]; function generateMockLogs(senderOffset: number) { const logs: { [k: string]: TxScopedL2Log[] } = {}; @@ -231,7 +231,7 @@ describe('Simulator oracle', () => { const partialAddress = Fr.random(); const address = computeAddress(keys.publicKeys, partialAddress); const completeAddress = new CompleteAddress(address, keys.publicKeys, partialAddress); - return { completeAddress, ivsk: keys.masterIncomingViewingSecretKey }; + return { completeAddress, ivsk: keys.masterIncomingViewingSecretKey, secretKey: new Fr(index) }; }); for (const sender of senders) { await database.addContactAddress(sender.completeAddress.address); @@ -267,6 +267,42 @@ describe('Simulator oracle', () => { expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 + SENDER_OFFSET_WINDOW_SIZE); }); + it('should sync tagged logs as senders', async () => { + for (const sender of senders) { + await database.addCompleteAddress(sender.completeAddress); + await keyStore.addAccount(sender.secretKey, sender.completeAddress.partialAddress); + } + + const senderOffset = 0; + generateMockLogs(senderOffset); + + // Recompute the secrets (as recipient) to ensure indexes are updated + const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address); + const secrets = senders.map(sender => { + const firstSenderSharedSecret = computeTaggingSecret(recipient, ivsk, sender.completeAddress.address); + return poseidon2Hash([firstSenderSharedSecret.x, firstSenderSharedSecret.y, contractAddress]); + }); + + const indexesAsSender = await database.getTaggingSecretsIndexesAsSender(secrets); + expect(indexesAsSender).toStrictEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(0); + + for (let i = 0; i < senders.length; i++) { + await simulatorOracle.syncTaggedLogsAsSender( + contractAddress, + senders[i].completeAddress.address, + recipient.address, + ); + } + + const indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); + expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); + + // We expect getLogsByTags to be called N + 1 times, where N is the index. + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 * 5 + 3 * 5); + }); + it('should sync tagged logs with a sender index offset', async () => { const senderOffset = 5; generateMockLogs(senderOffset); From 1a5cd0ba5fe52e5b0ac3cf72ac829624a9c9864a Mon Sep 17 00:00:00 2001 From: sklppy88 Date: Thu, 21 Nov 2024 09:22:11 +0000 Subject: [PATCH 3/5] batched approach --- .../pxe/src/simulator_oracle/index.ts | 34 +++++++++++-------- .../simulator_oracle/simulator_oracle.test.ts | 3 +- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 8d35b369682..a8085218b1e 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -350,23 +350,29 @@ export class SimulatorOracle implements DBOracle { const appTaggingSecret = await this.#calculateTaggingSecret(contractAddress, sender, recipient); let [currentIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]); - while (true) { - const currentIndexedAppTaggingSecret = new IndexedTaggingSecret(appTaggingSecret, currentIndex); - const currentTag = currentIndexedAppTaggingSecret.computeTag(recipient); + const WINDOW = 5; - const [[possibleLog]] = await this.aztecNode.getLogsByTags([currentTag]); + let possibleLogs: TxScopedL2Log[][]; - if (possibleLog !== undefined) { - currentIndex++; - } else { - this.log.debug( - `Syncing logs for sender ${sender}, secret ${appTaggingSecret}:${currentIndex} at contract: ${contractName}(${contractAddress})`, - ); + do { + const currentTags = [...new Array(WINDOW)].map((_, i) => { + const indexedAppTaggingSecret = new IndexedTaggingSecret(appTaggingSecret, currentIndex + i); + return indexedAppTaggingSecret.computeTag(recipient); + }); - await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, currentIndex)]); - break; - } - } + possibleLogs = await this.aztecNode.getLogsByTags(currentTags); + currentIndex += WINDOW; + } while (possibleLogs.every(log => log.length !== 0)); + + // We are getting the first empty index, and subtracting it from our WINDOW. So if our first empty index is 1, + // and our current index is 5, which means that we went through the loop one time, we take our current index, + // and subtract WINDOW - first empty index (= 5-1) from it. This means that new index will be 1. + const newIndex = currentIndex - (WINDOW - possibleLogs.findIndex(log => log.length === 0)); + + await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, newIndex)]); + this.log.debug( + `Syncing logs for sender ${sender}, secret ${appTaggingSecret}:${currentIndex} at contract: ${contractName}(${contractAddress})`, + ); } /** diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index 002fd32b7f4..bd7fb856242 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -299,8 +299,7 @@ describe('Simulator oracle', () => { const indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); - // We expect getLogsByTags to be called N + 1 times, where N is the index. - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 * 5 + 3 * 5); + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS); }); it('should sync tagged logs with a sender index offset', async () => { From 75c2b1cab2965fb112b00fe8071b3e3dc29a546f Mon Sep 17 00:00:00 2001 From: sklppy88 Date: Fri, 22 Nov 2024 14:42:08 +0000 Subject: [PATCH 4/5] adding efficient-ish sliding window --- .../pxe/src/simulator_oracle/index.ts | 42 ++++++++++++++----- .../simulator_oracle/simulator_oracle.test.ts | 24 +++++++++-- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index a8085218b1e..ffbc557984d 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -346,30 +346,50 @@ export class SimulatorOracle implements DBOracle { sender: AztecAddress, recipient: AztecAddress, ): Promise { - const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); const appTaggingSecret = await this.#calculateTaggingSecret(contractAddress, sender, recipient); let [currentIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]); - const WINDOW = 5; + const INDEX_OFFSET = 10; + + let previousEmptyBack = 0; + let currentEmptyBack = 0; + let currentEmptyFront: number; - let possibleLogs: TxScopedL2Log[][]; + // The below code is trying to find the index of the start of the first window in which for all elements of window, we do not see logs. + // We take our window size, and fetch the node for these logs. We store both the amount of empty consecutive slots from the front and the back. + // We use our current empty consecutive slots from the front, as well as the previous consecutive empty slots from the back to see if we ever hit a time where there + // is a window in which we see the combination of them to be greater than the window's size. If true, we rewind current index to the start of said window and use it. + // Assuming two windows of 5: + // [0, 1, 0, 0, 0], [0, 0, 0, 1, 1] + // We can see that when processing the second window, the previous amount of empty slots from the back of the window (3), added with the empty elements from the front of the window (3) + // is greater than 5 (6) and therefore we have found a window to use. + // We simply need to take the number of elements (10) - the size of the window (5) - the number of consecutive empty elements from the back of the last window (3) = 2; + // This is the first index of our desired window. do { - const currentTags = [...new Array(WINDOW)].map((_, i) => { + const currentTags = [...new Array(INDEX_OFFSET)].map((_, i) => { const indexedAppTaggingSecret = new IndexedTaggingSecret(appTaggingSecret, currentIndex + i); return indexedAppTaggingSecret.computeTag(recipient); }); + previousEmptyBack = currentEmptyBack; + + const possibleLogs = await this.aztecNode.getLogsByTags(currentTags); + + const indexOfFirstLog = possibleLogs.findIndex(possibleLog => possibleLog.length !== 0); + currentEmptyFront = indexOfFirstLog === -1 ? INDEX_OFFSET : indexOfFirstLog; - possibleLogs = await this.aztecNode.getLogsByTags(currentTags); - currentIndex += WINDOW; - } while (possibleLogs.every(log => log.length !== 0)); + const indexOfLastLog = possibleLogs.findLastIndex(possibleLog => possibleLog.length !== 0); + currentEmptyBack = indexOfLastLog === -1 ? INDEX_OFFSET : INDEX_OFFSET - 1 - indexOfLastLog; - // We are getting the first empty index, and subtracting it from our WINDOW. So if our first empty index is 1, - // and our current index is 5, which means that we went through the loop one time, we take our current index, - // and subtract WINDOW - first empty index (= 5-1) from it. This means that new index will be 1. - const newIndex = currentIndex - (WINDOW - possibleLogs.findIndex(log => log.length === 0)); + currentIndex += INDEX_OFFSET; + } while (currentEmptyFront + previousEmptyBack < INDEX_OFFSET); + + // We unwind the entire current window and the amount of consecutive empty slots from the previous window + const newIndex = currentIndex - (INDEX_OFFSET + previousEmptyBack); await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, newIndex)]); + + const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); this.log.debug( `Syncing logs for sender ${sender}, secret ${appTaggingSecret}:${currentIndex} at contract: ${contractName}(${contractAddress})`, ); diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index bd7fb856242..ad1dd07dd0d 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -273,7 +273,7 @@ describe('Simulator oracle', () => { await keyStore.addAccount(sender.secretKey, sender.completeAddress.partialAddress); } - const senderOffset = 0; + let senderOffset = 0; generateMockLogs(senderOffset); // Recompute the secrets (as recipient) to ensure indexes are updated @@ -296,10 +296,28 @@ describe('Simulator oracle', () => { ); } - const indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); + let indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS); + // Two windows are fetch for each sender + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2); + aztecNode.getLogsByTags.mockReset(); + + // We add more logs at the end of the window to make sure we only detect them and bump the indexes if it lies within our window + senderOffset = 11; + generateMockLogs(senderOffset); + for (let i = 0; i < senders.length; i++) { + await simulatorOracle.syncTaggedLogsAsSender( + contractAddress, + senders[i].completeAddress.address, + recipient.address, + ); + } + + indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); + expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 13, 13, 13, 13, 13]); + + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2); }); it('should sync tagged logs with a sender index offset', async () => { From 4a8ae3e20f80011bf6dad0766b237f5a87f6dfd2 Mon Sep 17 00:00:00 2001 From: sklppy88 Date: Fri, 22 Nov 2024 15:54:51 +0000 Subject: [PATCH 5/5] fix comments --- yarn-project/pxe/src/simulator_oracle/index.ts | 9 ++++++--- .../pxe/src/simulator_oracle/simulator_oracle.test.ts | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index ffbc557984d..a7ca08660f2 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -360,11 +360,14 @@ export class SimulatorOracle implements DBOracle { // We use our current empty consecutive slots from the front, as well as the previous consecutive empty slots from the back to see if we ever hit a time where there // is a window in which we see the combination of them to be greater than the window's size. If true, we rewind current index to the start of said window and use it. // Assuming two windows of 5: - // [0, 1, 0, 0, 0], [0, 0, 0, 1, 1] - // We can see that when processing the second window, the previous amount of empty slots from the back of the window (3), added with the empty elements from the front of the window (3) + // [0, 1, 0, 1, 0], [0, 0, 0, 0, 0] + // We can see that when processing the second window, the previous amount of empty slots from the back of the window (1), added with the empty elements from the front of the window (5) // is greater than 5 (6) and therefore we have found a window to use. - // We simply need to take the number of elements (10) - the size of the window (5) - the number of consecutive empty elements from the back of the last window (3) = 2; + // We simply need to take the number of elements (10) - the size of the window (5) - the number of consecutive empty elements from the back of the last window (1) = 4; // This is the first index of our desired window. + // Note that if we ever see a situation like so: + // [0, 1, 0, 1, 0], [0, 0, 0, 0, 1] + // This also returns the correct index (4), but this is indicative of a problem / desync. i.e. we should never have a window that has a log that exists after the window. do { const currentTags = [...new Array(INDEX_OFFSET)].map((_, i) => { diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index ad1dd07dd0d..388253c9005 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -304,7 +304,7 @@ describe('Simulator oracle', () => { aztecNode.getLogsByTags.mockReset(); // We add more logs at the end of the window to make sure we only detect them and bump the indexes if it lies within our window - senderOffset = 11; + senderOffset = 10; generateMockLogs(senderOffset); for (let i = 0; i < senders.length; i++) { await simulatorOracle.syncTaggedLogsAsSender( @@ -315,7 +315,7 @@ describe('Simulator oracle', () => { } indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets); - expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 13, 13, 13, 13, 13]); + expect(indexesAsSenderAfterSync).toStrictEqual([11, 11, 11, 11, 11, 12, 12, 12, 12, 12]); expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2); });