Merge remote-tracking branch 'origin/mw/blob-circuit' into mw/blob-ci…

…rcuit-3

.github/spot-runner-action/src/main.ts

@@ -23,10 +23,11 @@ async function pollSpotStatus(
     }
     try {
       core.info("Found ec2 instance, looking for runners.");
-      if (process.env.WAIT_FOR_RUNNERS === "false" || await ghClient.hasRunner([config.githubJobId])) {
+      // TODO find out whatever happened here but we seem to not be able to wait for runners
+      //if (process.env.WAIT_FOR_RUNNERS === "false" || await ghClient.hasRunner([config.githubJobId])) {
         // we have runners
         return instances[0].InstanceId!;
-      }
+      //}
     } catch (err) {}
     // wait 10 seconds
     await new Promise((r) => setTimeout(r, 10000));
@@ -174,11 +175,11 @@ async function startWithGithubRunners(config: ActionConfig) {
       return false;
     }
     await setupGithubRunners(ip, config);
-    if (instanceId) await ghClient.pollForRunnerCreation([config.githubJobId]);
-    else {
-      core.error("Instance failed to register with Github Actions");
-      throw Error("Instance failed to register with Github Actions");
-    }
+    // if (instanceId) await ghClient.pollForRunnerCreation([config.githubJobId]);
+    // else {
+    //  core.error("Instance failed to register with Github Actions");
+    //  throw Error("Instance failed to register with Github Actions");
+    // }
     core.info("Done setting up runner.")
   }
   // Export to github environment

docs/docs/guides/developer_guides/smart_contracts/profiling_transactions.md

@@ -0,0 +1,71 @@
+---
+title: Profiling Transactions
+sidebar_position: 5
+tags: [contracts, profiling]
+---
+
+# Profiling Transactions
+
+An Aztec transaction typically consists of a private and a public part. The private part is where the user executes contract logic within the PXE and generates a proof of execution, which is then sent to the sequencer.
+
+Since proof generation is an expensive operation that needs to be done on the client side, it is important to optimize the private contract logic. It is desirable to keep the gate count of circuits representing the private contract logic as low as possible.
+
+A private transaction can involve multiple function calls. It starts with an account `entrypoint()` which may call several private functions to execute the application logic, which in turn might call other functions. Moreover, every private function call has to go through a round of kernel circuits. Read more about the transaction lifecycle [here](../../../aztec/concepts/transactions.md).
+
+In this guide, we will look at how to profile the private execution of a transaction, allowing you to get the gate count of each private function within the transaction, including the kernel circuits.
+
+## Prerequisites
+
+- `aztec-nargo` installed (go to [Sandbox section](../../../reference/developer_references/sandbox_reference/sandbox-reference.md) for installation instructions)
+- `aztec-wallet` installed (installed as part of the Sandbox)
+- Aztec Sandbox running with **proving enabled** (go to [Sandbox PXE Proving](../local_env/sandbox_proving.md) for instructions)
+
+## Profiling using aztec-wallet
+
+The profiling tool is integrated into the `aztec-wallet`.
+
+In this example, we will profile a simple "private token transfer" transaction which uses the [transfer](https://github.com/AztecProtocol/aztec-packages/blob/master/noir-projects/noir-contracts/contracts/token_contract/src/main.nr#L269) method in the token contract.
+If you want to follow along, you'll need to clone the Aztec [monorepo](https://github.com/AztecProtocol/aztec-packages) and [compile](./how_to_compile_contract.md) the `token_contract` in `noir-projects/noir-contracts` by running `aztec-nargo compile --package token_contract`.
+
+Let's deploy the necessary account and token contracts first:
+
+```bash
+# Deploy accounts
+aztec-wallet create-account -a owner
+aztec-wallet create-account -a user
+
+# Deploy a token contract and mint 100 tokens to the user
+aztec-wallet deploy token_contract@Token --args accounts:owner Test TST 18 -f owner -a token
+aztec-wallet send mint_to_private -ca token --args accounts:owner accounts:user 100 -f owner
+```
+
+Now, the `user` can transfer tokens by running:
+
+```bash
+# Send the tokens back to the owner
+aztec-wallet send transfer -ca token --args accounts:owner 40 -f user
+```
+
+Instead of sending the above transaction, you can simulate it by running the `simulate` command with the same parameters, and then add a `--profile` flag to profile the gate count of each private function in the transaction.
+
+```bash
+aztec-wallet simulate --profile transfer -ca token --args accounts:owner 40 -f user
+```
+
+This will print the following results after some time:
+
+```bash
+Gate count per circuit:
+   SchnorrAccount:entrypoint                          Gates: 26,363     Acc: 26,363
+   private_kernel_init                                Gates: 34,887     Acc: 61,250
+   Token:transfer                                     Gates: 28,229     Acc: 89,479
+   private_kernel_inner                               Gates: 57,530     Acc: 147,009
+   private_kernel_reset                               Gates: 86,600     Acc: 233,609
+   private_kernel_tail                                Gates: 13,045     Acc: 246,654
+
+Total gates: 246,654
+```
+
+Here you can see the gate count of each private function call in the transaction along with the kernel circuits needed in between, and the total gate count.
+
+This will help you understand which parts of your transaction are bottlenecks and optimize the contract logic accordingly.

docs/docs/reference/developer_references/sandbox_reference/cli_wallet_reference.md

@@ -130,6 +130,16 @@ Simulates a transaction instead of sending it. This allows you to obtain i.e. th
 aztec-wallet simulate --from master_yoda --contract-address jedi_order --args "luke_skywalker" train_jedi
 ```
 
+### Profile
+
+Simulates a transaction with profiling enabled. This allows you to get the gate count of each private function in the transaction. Read more about profiling [here](../../../guides/developer_guides/smart_contracts/profiling_transactions.md).
+
+#### Example
+
+```bash
+aztec-wallet simulate --profile --from master_yoda --contract-address jedi_order --args "luke_skywalker" train_jedi
+```
+
 ### Bridge Fee Juice
 
 The wallet provides an easy way to mint the fee-paying asset on L1 and bridging it to L2. We call it Fee Juice and you can read more about it in the [protocol specs](../../../protocol-specs/gas-and-fees/fee-juice.md).

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -71,7 +71,7 @@ import {
 } from '@aztec/p2p';
 import { ProtocolContractAddress } from '@aztec/protocol-contracts';
 import { GlobalVariableBuilder, SequencerClient } from '@aztec/sequencer-client';
-import { PublicProcessorFactory, WASMSimulator, createSimulationProvider } from '@aztec/simulator';
+import { PublicProcessorFactory, createSimulationProvider } from '@aztec/simulator';
 import { type TelemetryClient } from '@aztec/telemetry-client';
 import { NoopTelemetryClient } from '@aztec/telemetry-client/noop';
 import { createValidatorClient } from '@aztec/validator-client';
@@ -733,11 +733,7 @@ export class AztecNodeService implements AztecNode {
       feeRecipient,
     );
     const prevHeader = (await this.blockSource.getBlock(-1))?.header;
-    const publicProcessorFactory = new PublicProcessorFactory(
-      this.contractDataSource,
-      new WASMSimulator(),
-      this.telemetry,
-    );
+    const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.telemetry);
 
     const fork = await this.worldStateSynchronizer.fork();
 

yarn-project/circuit-types/src/tx/tx.ts

@@ -2,7 +2,8 @@ import {
   ClientIvcProof,
   ContractClassRegisteredEvent,
   PrivateKernelTailCircuitPublicInputs,
-  type PublicKernelCircuitPublicInputs,
+  type PrivateToPublicAccumulatedData,
+  type ScopedLogHash,
 } from '@aztec/circuits.js';
 import { type Buffer32 } from '@aztec/foundation/buffer';
 import { arraySerializedSizeOfNonEmpty } from '@aztec/foundation/collection';
@@ -344,29 +345,26 @@ export class Tx extends Gossipable {
    * @param logHashes the individual log hashes we want to keep
    * @param out the output to put passing logs in, to keep this function abstract
    */
-  public filterRevertedLogs(kernelOutput: PublicKernelCircuitPublicInputs) {
+  public filterRevertedLogs(
+    privateNonRevertible: PrivateToPublicAccumulatedData,
+    unencryptedLogsHashes: ScopedLogHash[],
+  ) {
     this.encryptedLogs = this.encryptedLogs.filterScoped(
-      kernelOutput.endNonRevertibleData.encryptedLogsHashes,
+      privateNonRevertible.encryptedLogsHashes,
       EncryptedTxL2Logs.empty(),
     );
 
-    this.unencryptedLogs = this.unencryptedLogs.filterScoped(
-      kernelOutput.endNonRevertibleData.unencryptedLogsHashes,
-      UnencryptedTxL2Logs.empty(),
-    );
-
     this.noteEncryptedLogs = this.noteEncryptedLogs.filter(
-      kernelOutput.endNonRevertibleData.noteEncryptedLogsHashes,
+      privateNonRevertible.noteEncryptedLogsHashes,
       EncryptedNoteTxL2Logs.empty(),
     );
 
-    // See comment in enqueued_calls_processor.ts -> tx.filterRevertedLogs()
-    if (this.data.forPublic) {
-      this.contractClassLogs = this.contractClassLogs.filterScoped(
-        this.data.forPublic?.nonRevertibleAccumulatedData.contractClassLogsHashes,
-        ContractClassTxL2Logs.empty(),
-      );
-    }
+    this.contractClassLogs = this.contractClassLogs.filterScoped(
+      privateNonRevertible.contractClassLogsHashes,
+      ContractClassTxL2Logs.empty(),
+    );
+
+    this.unencryptedLogs = this.unencryptedLogs.filterScoped(unencryptedLogsHashes, UnencryptedTxL2Logs.empty());
   }
 }
 

yarn-project/prover-client/src/mocks/test_context.ts

@@ -16,7 +16,6 @@ import {
   PublicExecutionResultBuilder,
   type PublicExecutor,
   PublicProcessor,
-  RealPublicKernelCircuitSimulator,
   type SimulationProvider,
   WASMSimulator,
   type WorldStateDB,
@@ -68,7 +67,6 @@ export class TestContext {
 
     const publicExecutor = mock<PublicExecutor>();
     const worldStateDB = mock<WorldStateDB>();
-    const publicKernel = new RealPublicKernelCircuitSimulator(new WASMSimulator());
     const telemetry = new NoopTelemetryClient();
 
     // Separated dbs for public processor and prover - see public_processor for context
@@ -88,7 +86,6 @@ export class TestContext {
     const processor = PublicProcessor.create(
       publicDb,
       publicExecutor,
-      publicKernel,
       globalVariables,
       Header.empty(),
       worldStateDB,

yarn-project/prover-node/src/prover-node.ts

@@ -56,7 +56,7 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr
     private readonly contractDataSource: ContractDataSource,
     private readonly worldState: WorldStateSynchronizer,
     private readonly coordination: ProverCoordination & Maybe<Service>,
-    private readonly simulator: SimulationProvider,
+    private readonly _simulator: SimulationProvider,
     private readonly quoteProvider: QuoteProvider,
     private readonly quoteSigner: QuoteSigner,
     private readonly claimsMonitor: ClaimsMonitor,
@@ -243,11 +243,7 @@ export class ProverNode implements ClaimsMonitorHandler, EpochMonitorHandler, Pr
     const proverDb = await this.worldState.fork(fromBlock - 1);
 
     // Create a processor using the forked world state
-    const publicProcessorFactory = new PublicProcessorFactory(
-      this.contractDataSource,
-      this.simulator,
-      this.telemetryClient,
-    );
+    const publicProcessorFactory = new PublicProcessorFactory(this.contractDataSource, this.telemetryClient);
 
     const cleanUp = async () => {
       await publicDb.close();

yarn-project/sequencer-client/src/client/sequencer-client.ts

@@ -40,13 +40,13 @@ export class SequencerClient {
     contractDataSource: ContractDataSource,
     l2BlockSource: L2BlockSource,
     l1ToL2MessageSource: L1ToL2MessageSource,
-    simulationProvider: SimulationProvider,
+    _simulationProvider: SimulationProvider,
     telemetryClient: TelemetryClient,
   ) {
     const publisher = new L1Publisher(config, telemetryClient);
     const globalsBuilder = new GlobalVariableBuilder(config);
 
-    const publicProcessorFactory = new PublicProcessorFactory(contractDataSource, simulationProvider, telemetryClient);
+    const publicProcessorFactory = new PublicProcessorFactory(contractDataSource, telemetryClient);
 
     const rollup = publisher.getRollupContract();
     const [l1GenesisTime, slotDuration] = await Promise.all([

yarn-project/simulator/src/avm/avm_simulator.test.ts

@@ -138,7 +138,7 @@ describe('AVM simulator: transpiled Noir contracts', () => {
     const contractClass = makeContractClassPublic(0, publicFn);
     const contractInstance = makeContractInstanceFromClassId(contractClass.id);
 
-    // The values here should match those in `avm_simulator.test.ts`
+    // The values here should match those in getContractInstance test case
     const instanceGet = new SerializableContractInstance({
       version: 1,
       salt: new Fr(0x123),

yarn-project/simulator/src/avm/journal/journal.ts

@@ -39,6 +39,9 @@ export class AvmPersistableStateManager {
   /** Interface to perform merkle tree operations */
   public merkleTrees: MerkleTreeWriteOperations;
 
+  /** Make sure a forked state is never merged twice. */
+  private alreadyMergedIntoParent = false;
+
   constructor(
     /** Reference to node storage */
     private readonly worldStateDB: WorldStateDB,
@@ -79,16 +82,46 @@ export class AvmPersistableStateManager {
   /**
    * Create a new state manager forked from this one
    */
-  public fork(incrementSideEffectCounter: boolean = false) {
+  public fork() {
     return new AvmPersistableStateManager(
       this.worldStateDB,
-      this.trace.fork(incrementSideEffectCounter),
+      this.trace.fork(),
       this.publicStorage.fork(),
       this.nullifiers.fork(),
       this.doMerkleOperations,
     );
   }
 
+  /**
+   * Accept forked world state modifications & traced side effects / hints
+   */
+  public merge(forkedState: AvmPersistableStateManager) {
+    this._merge(forkedState, /*reverted=*/ false);
+  }
+
+  /**
+   * Reject forked world state modifications & traced side effects, keep traced hints
+   */
+  public reject(forkedState: AvmPersistableStateManager) {
+    this._merge(forkedState, /*reverted=*/ true);
+  }
+
+  /**
+   * Commit cached storage writes to the DB.
+   * Keeps public storage up to date from tx to tx within a block.
+   */
+  public async commitStorageWritesToDB() {
+    await this.publicStorage.commitToDB();
+  }
+
+  private _merge(forkedState: AvmPersistableStateManager, reverted: boolean) {
+    // sanity check to avoid merging the same forked trace twice
+    assert(!this.alreadyMergedIntoParent, 'Cannot merge forked state that has already been merged into its parent!');
+    this.publicStorage.acceptAndMerge(forkedState.publicStorage);
+    this.nullifiers.acceptAndMerge(forkedState.nullifiers);
+    this.trace.merge(forkedState.trace, reverted);
+  }
+
   /**
    * Write to public storage, journal/trace the write.
    *
@@ -427,21 +460,6 @@ export class AvmPersistableStateManager {
     }
   }
 
-  /**
-   * Accept nested world state modifications
-   */
-  public mergeForkedState(forkedState: AvmPersistableStateManager) {
-    this.publicStorage.acceptAndMerge(forkedState.publicStorage);
-    this.nullifiers.acceptAndMerge(forkedState.nullifiers);
-    this.trace.mergeSuccessfulForkedTrace(forkedState.trace);
-  }
-
-  public rejectForkedState(forkedState: AvmPersistableStateManager) {
-    this.publicStorage.acceptAndMerge(forkedState.publicStorage);
-    this.nullifiers.acceptAndMerge(forkedState.nullifiers);
-    this.trace.mergeRevertedForkedTrace(forkedState.trace);
-  }
-
   /**
    * Get a contract's bytecode from the contracts DB, also trace the contract class and instance
    */

yarn-project/simulator/src/avm/opcodes/external_calls.ts

@@ -97,9 +97,11 @@ abstract class ExternalCall extends Instruction {
     // Refund unused gas
     context.machineState.refundGas(gasLeftToGas(nestedContext.machineState));
 
-    // Accept the nested call's state and trace the nested call
+    // Merge nested call's state and trace based on whether it succeeded.
     if (success) {
-      context.persistableState.mergeForkedState(nestedContext.persistableState);
+      context.persistableState.merge(nestedContext.persistableState);
+    } else {
+      context.persistableState.reject(nestedContext.persistableState);
     }
     await context.persistableState.traceNestedCall(
       /*nestedState=*/ nestedContext.persistableState,

yarn-project/simulator/src/public/dual_side_effect_trace.ts

@@ -27,11 +27,12 @@ export class DualSideEffectTrace implements PublicSideEffectTraceInterface {
     public readonly enqueuedCallTrace: PublicEnqueuedCallSideEffectTrace,
   ) {}
 
-  public fork(incrementSideEffectCounter: boolean = false) {
-    return new DualSideEffectTrace(
-      this.innerCallTrace.fork(incrementSideEffectCounter),
-      this.enqueuedCallTrace.fork(incrementSideEffectCounter),
-    );
+  public fork() {
+    return new DualSideEffectTrace(this.innerCallTrace.fork(), this.enqueuedCallTrace.fork());
+  }
+
+  public merge(nestedTrace: this, reverted: boolean = false) {
+    this.enqueuedCallTrace.merge(nestedTrace.enqueuedCallTrace, reverted);
   }
 
   public getCounter() {
@@ -232,14 +233,6 @@ export class DualSideEffectTrace implements PublicSideEffectTraceInterface {
     this.enqueuedCallTrace.traceEnqueuedCall(publicCallRequest, calldata, reverted);
   }
 
-  public mergeSuccessfulForkedTrace(nestedTrace: this) {
-    this.enqueuedCallTrace.mergeSuccessfulForkedTrace(nestedTrace.enqueuedCallTrace);
-  }
-
-  public mergeRevertedForkedTrace(nestedTrace: this) {
-    this.enqueuedCallTrace.mergeRevertedForkedTrace(nestedTrace.enqueuedCallTrace);
-  }
-
   /**
    * Convert this trace to a PublicExecutionResult for use externally to the simulator.
    */