feat: sync tags as sender

yarn-project/pxe/src/database/kv_pxe_database.ts

@@ -668,25 +668,19 @@ export class KVPxeDatabase implements PxeDatabase {
     return incomingNotesSize + outgoingNotesSize + treeRootsSize + authWitsSize + addressesSize;
   }
 
-  async incrementTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise<void> {
-    await this.#incrementTaggingSecretsIndexes(appTaggingSecrets, this.#taggingSecretIndexesForSenders);
+  async setTaggingSecretsIndexesAsSender(indexedSecrets: IndexedTaggingSecret[]): Promise<void> {
+    await this.#setTaggingSecretsIndexes(indexedSecrets, this.#taggingSecretIndexesForSenders);
   }
 
-  async #incrementTaggingSecretsIndexes(appTaggingSecrets: Fr[], storageMap: AztecMap<string, number>): Promise<void> {
-    const indexes = await this.#getTaggingSecretsIndexes(appTaggingSecrets, storageMap);
-    await this.db.transaction(() => {
-      indexes.forEach((taggingSecretIndex, listIndex) => {
-        const nextIndex = taggingSecretIndex + 1;
-        void storageMap.set(appTaggingSecrets[listIndex].toString(), nextIndex);
-      });
-    });
+  async setTaggingSecretsIndexesAsRecipient(indexedSecrets: IndexedTaggingSecret[]): Promise<void> {
+    await this.#setTaggingSecretsIndexes(indexedSecrets, this.#taggingSecretIndexesForRecipients);
   }
 
-  async setTaggingSecretsIndexesAsRecipient(indexedSecrets: IndexedTaggingSecret[]): Promise<void> {
-    await this.db.transaction(() => {
-      indexedSecrets.forEach(indexedSecret => {
-        void this.#taggingSecretIndexesForRecipients.set(indexedSecret.secret.toString(), indexedSecret.index);
-      });
+  #setTaggingSecretsIndexes(indexedSecrets: IndexedTaggingSecret[], storageMap: AztecMap<string, number>) {
+    return this.db.transaction(() => {
+      indexedSecrets.forEach(
+        indexedSecret => void storageMap.set(indexedSecret.secret.toString(), indexedSecret.index),
+      );
     });
   }
 

yarn-project/pxe/src/database/pxe_database.ts

@@ -202,11 +202,11 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD
   getTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise<number[]>;
 
   /**
-   * Increments the index for the provided app siloed tagging secrets in the senders database
-   * To be used when the generated tags have been used as sender
+   * Sets the index for the provided app siloed tagging secrets
+   * To be used when the generated tags have been "seen" as a sender
    * @param appTaggingSecrets - The app siloed tagging secrets.
    */
-  incrementTaggingSecretsIndexesAsSender(appTaggingSecrets: Fr[]): Promise<void>;
+  setTaggingSecretsIndexesAsSender(indexedTaggingSecrets: IndexedTaggingSecret[]): Promise<void>;
 
   /**
    * Sets the index for the provided app siloed tagging secrets

yarn-project/pxe/src/simulator_oracle/index.ts

@@ -268,8 +268,11 @@ export class SimulatorOracle implements DBOracle {
     sender: AztecAddress,
     recipient: AztecAddress,
   ): Promise<IndexedTaggingSecret> {
+    await this.syncTaggedLogsAsSender(contractAddress, sender, recipient);
+
     const secret = await this.#calculateTaggingSecret(contractAddress, sender, recipient);
     const [index] = await this.db.getTaggingSecretsIndexesAsSender([secret]);
+
     return new IndexedTaggingSecret(secret, index);
   }
 
@@ -289,7 +292,9 @@ export class SimulatorOracle implements DBOracle {
     this.log.verbose(
       `Incrementing secret ${secret} as sender ${sender} for recipient: ${recipient} at contract: ${contractName}(${contractAddress})`,
     );
-    await this.db.incrementTaggingSecretsIndexesAsSender([secret]);
+
+    const [index] = await this.db.getTaggingSecretsIndexesAsSender([secret]);
+    await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(secret, index + 1)]);
   }
 
   async #calculateTaggingSecret(contractAddress: AztecAddress, sender: AztecAddress, recipient: AztecAddress) {
@@ -329,6 +334,70 @@ export class SimulatorOracle implements DBOracle {
     return appTaggingSecrets.map((secret, i) => new IndexedTaggingSecret(secret, indexes[i]));
   }
 
+  /**
+   * Updates the local index of the shared tagging secret of a sender / recipient pair
+   * if a log with a larger index is found from the node.
+   * @param contractAddress - The address of the contract that the logs are tagged for
+   * @param sender - The address of the sender, we must know the sender's ivsk_m.
+   * @param recipient - The address of the recipient.
+   */
+  public async syncTaggedLogsAsSender(
+    contractAddress: AztecAddress,
+    sender: AztecAddress,
+    recipient: AztecAddress,
+  ): Promise<void> {
+    const appTaggingSecret = await this.#calculateTaggingSecret(contractAddress, sender, recipient);
+    let [currentIndex] = await this.db.getTaggingSecretsIndexesAsSender([appTaggingSecret]);
+
+    const INDEX_OFFSET = 10;
+
+    let previousEmptyBack = 0;
+    let currentEmptyBack = 0;
+    let currentEmptyFront: number;
+
+    // The below code is trying to find the index of the start of the first window in which for all elements of window, we do not see logs.
+    // We take our window size, and fetch the node for these logs. We store both the amount of empty consecutive slots from the front and the back.
+    // We use our current empty consecutive slots from the front, as well as the previous consecutive empty slots from the back to see if we ever hit a time where there
+    // is a window in which we see the combination of them to be greater than the window's size. If true, we rewind current index to the start of said window and use it.
+    // Assuming two windows of 5:
+    // [0, 1, 0, 1, 0], [0, 0, 0, 0, 0]
+    // We can see that when processing the second window, the previous amount of empty slots from the back of the window (1), added with the empty elements from the front of the window (5)
+    // is greater than 5 (6) and therefore we have found a window to use.
+    // We simply need to take the number of elements (10) - the size of the window (5) - the number of consecutive empty elements from the back of the last window (1) = 4;
+    // This is the first index of our desired window.
+    // Note that if we ever see a situation like so:
+    // [0, 1, 0, 1, 0], [0, 0, 0, 0, 1]
+    // This also returns the correct index (4), but this is indicative of a problem / desync. i.e. we should never have a window that has a log that exists after the window.
+
+    do {
+      const currentTags = [...new Array(INDEX_OFFSET)].map((_, i) => {
+        const indexedAppTaggingSecret = new IndexedTaggingSecret(appTaggingSecret, currentIndex + i);
+        return indexedAppTaggingSecret.computeTag(recipient);
+      });
+      previousEmptyBack = currentEmptyBack;
+
+      const possibleLogs = await this.aztecNode.getLogsByTags(currentTags);
+
+      const indexOfFirstLog = possibleLogs.findIndex(possibleLog => possibleLog.length !== 0);
+      currentEmptyFront = indexOfFirstLog === -1 ? INDEX_OFFSET : indexOfFirstLog;
+
+      const indexOfLastLog = possibleLogs.findLastIndex(possibleLog => possibleLog.length !== 0);
+      currentEmptyBack = indexOfLastLog === -1 ? INDEX_OFFSET : INDEX_OFFSET - 1 - indexOfLastLog;
+
+      currentIndex += INDEX_OFFSET;
+    } while (currentEmptyFront + previousEmptyBack < INDEX_OFFSET);
+
+    // We unwind the entire current window and the amount of consecutive empty slots from the previous window
+    const newIndex = currentIndex - (INDEX_OFFSET + previousEmptyBack);
+
+    await this.db.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appTaggingSecret, newIndex)]);
+
+    const contractName = await this.contractDataOracle.getDebugContractName(contractAddress);
+    this.log.debug(
+      `Syncing logs for sender ${sender}, secret ${appTaggingSecret}:${currentIndex} at contract: ${contractName}(${contractAddress})`,
+    );
+  }
+
   /**
    * Synchronizes the logs tagged with scoped addresses and all the senders in the addressbook.
    * Returns the unsynched logs and updates the indexes of the secrets used to tag them until there are no more logs to sync.

yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts

@@ -143,7 +143,7 @@ describe('Simulator oracle', () => {
   describe('sync tagged logs', () => {
     const NUM_SENDERS = 10;
     const SENDER_OFFSET_WINDOW_SIZE = 10;
-    let senders: { completeAddress: CompleteAddress; ivsk: Fq }[];
+    let senders: { completeAddress: CompleteAddress; ivsk: Fq; secretKey: Fr }[];
 
     function generateMockLogs(senderOffset: number) {
       const logs: { [k: string]: TxScopedL2Log[] } = {};
@@ -231,7 +231,7 @@ describe('Simulator oracle', () => {
         const partialAddress = Fr.random();
         const address = computeAddress(keys.publicKeys, partialAddress);
         const completeAddress = new CompleteAddress(address, keys.publicKeys, partialAddress);
-        return { completeAddress, ivsk: keys.masterIncomingViewingSecretKey };
+        return { completeAddress, ivsk: keys.masterIncomingViewingSecretKey, secretKey: new Fr(index) };
       });
       for (const sender of senders) {
         await database.addContactAddress(sender.completeAddress.address);
@@ -267,6 +267,59 @@ describe('Simulator oracle', () => {
       expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 + SENDER_OFFSET_WINDOW_SIZE);
     });
 
+    it('should sync tagged logs as senders', async () => {
+      for (const sender of senders) {
+        await database.addCompleteAddress(sender.completeAddress);
+        await keyStore.addAccount(sender.secretKey, sender.completeAddress.partialAddress);
+      }
+
+      let senderOffset = 0;
+      generateMockLogs(senderOffset);
+
+      // Recompute the secrets (as recipient) to ensure indexes are updated
+      const ivsk = await keyStore.getMasterIncomingViewingSecretKey(recipient.address);
+      const secrets = senders.map(sender => {
+        const firstSenderSharedSecret = computeTaggingSecret(recipient, ivsk, sender.completeAddress.address);
+        return poseidon2Hash([firstSenderSharedSecret.x, firstSenderSharedSecret.y, contractAddress]);
+      });
+
+      const indexesAsSender = await database.getTaggingSecretsIndexesAsSender(secrets);
+      expect(indexesAsSender).toStrictEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
+
+      expect(aztecNode.getLogsByTags.mock.calls.length).toBe(0);
+
+      for (let i = 0; i < senders.length; i++) {
+        await simulatorOracle.syncTaggedLogsAsSender(
+          contractAddress,
+          senders[i].completeAddress.address,
+          recipient.address,
+        );
+      }
+
+      let indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets);
+      expect(indexesAsSenderAfterSync).toStrictEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]);
+
+      // Two windows are fetch for each sender
+      expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2);
+      aztecNode.getLogsByTags.mockReset();
+
+      // We add more logs at the end of the window to make sure we only detect them and bump the indexes if it lies within our window
+      senderOffset = 10;
+      generateMockLogs(senderOffset);
+      for (let i = 0; i < senders.length; i++) {
+        await simulatorOracle.syncTaggedLogsAsSender(
+          contractAddress,
+          senders[i].completeAddress.address,
+          recipient.address,
+        );
+      }
+
+      indexesAsSenderAfterSync = await database.getTaggingSecretsIndexesAsSender(secrets);
+      expect(indexesAsSenderAfterSync).toStrictEqual([11, 11, 11, 11, 11, 12, 12, 12, 12, 12]);
+
+      expect(aztecNode.getLogsByTags.mock.calls.length).toBe(NUM_SENDERS * 2);
+    });
+
     it('should sync tagged logs with a sender index offset', async () => {
       const senderOffset = 5;
       generateMockLogs(senderOffset);

yarn-project/txe/src/oracle/txe_oracle.ts

@@ -773,8 +773,9 @@ export class TXE implements TypedOracle {
   }
 
   async incrementAppTaggingSecretIndexAsSender(sender: AztecAddress, recipient: AztecAddress): Promise<void> {
-    const directionalSecret = await this.#calculateTaggingSecret(this.contractAddress, sender, recipient);
-    await this.txeDatabase.incrementTaggingSecretsIndexesAsSender([directionalSecret]);
+    const appSecret = await this.#calculateTaggingSecret(this.contractAddress, sender, recipient);
+    const [index] = await this.txeDatabase.getTaggingSecretsIndexesAsSender([appSecret]);
+    await this.txeDatabase.setTaggingSecretsIndexesAsSender([new IndexedTaggingSecret(appSecret, index + 1)]);
   }
 
   async getAppTaggingSecretAsSender(sender: AztecAddress, recipient: AztecAddress): Promise<IndexedTaggingSecret> {