feat(avm): gzip avm bytecode (#6475)

We can use this to reduce bytecode size (just as Brillig does) until we
do bytecode validation.

```
AvmAcvmInteropTest::constant_field_avm: compressed 139 to 95 bytes
AvmAcvmInteropTest::call_acvm_from_avm: compressed 6343 to 1486 bytes
AvmAcvmInteropTest::avm_to_acvm_call: compressed 2145 to 552 bytes
AvmAcvmInteropTest::new_nullifier: compressed 91 to 74 bytes
AvmAcvmInteropTest::test_authwit_send_money: compressed 9090 to 1977 bytes
AvmInitializerTest::read_storage_immutable: compressed 2078 to 560 bytes
AvmInitializerTest::constructor: compressed 73769 to 6638 bytes
AvmNestedCallsTest::set_storage_single: compressed 719 to 201 bytes
AvmNestedCallsTest::nested_call_to_add_with_gas: compressed 48796 to 3976 bytes
AvmNestedCallsTest::assert_same: compressed 1758 to 452 bytes
AvmNestedCallsTest::new_nullifier: compressed 91 to 74 bytes
AvmNestedCallsTest::nested_static_call_to_add: compressed 48643 to 3871 bytes
AvmNestedCallsTest::nested_static_call_to_set_storage: compressed 47167 to 3573 bytes
AvmNestedCallsTest::add_args_return: compressed 140 to 92 bytes
AvmNestedCallsTest::nested_call_to_add: compressed 48258 to 3843 bytes
AvmNestedCallsTest::create_same_nullifier_in_nested_call: compressed 46760 to 3527 bytes
AvmNestedCallsTest::create_different_nullifier_in_nested_call: compressed 46809 to 3540 bytes
AvmTest::get_chain_id: compressed 111 to 81 bytes
AvmTest::to_radix_le: compressed 41037 to 2164 bytes
AvmTest::read_storage_list: compressed 1186 to 345 bytes
AvmTest::set_storage_map: compressed 5335 to 930 bytes
AvmTest::new_nullifier: compressed 91 to 74 bytes
AvmTest::set_opcode_u32: compressed 116 to 83 bytes
AvmTest::assertion_failure: compressed 2107 to 495 bytes
AvmTest::set_opcode_small_field: compressed 139 to 106 bytes
AvmTest::read_storage_map: compressed 4987 to 890 bytes
AvmTest::get_address: compressed 111 to 81 bytes
AvmTest::pedersen_hash_with_index: compressed 199 to 114 bytes
AvmTest::get_sender: compressed 111 to 81 bytes
AvmTest::add_storage_map: compressed 10417 to 1742 bytes
AvmTest::set_storage_list: compressed 864 to 219 bytes
AvmTest::get_fee_per_da_gas: compressed 111 to 81 bytes
AvmTest::u128_addition_overflow: compressed 46973 to 2808 bytes
AvmTest::read_storage_single: compressed 728 to 227 bytes
AvmTest::test_get_contract_instance_raw: compressed 1576 to 442 bytes
AvmTest::test_get_contract_instance: compressed 44797 to 2953 bytes
AvmTest::get_version: compressed 111 to 80 bytes
AvmTest::check_selector: compressed 44567 to 3039 bytes
AvmTest::get_transaction_fee: compressed 111 to 81 bytes
AvmTest::add_u128: compressed 2390 to 593 bytes
AvmTest::nullifier_collision: compressed 97 to 76 bytes
AvmTest::pedersen_hash: compressed 199 to 113 bytes
AvmTest::modulo2: compressed 177 to 104 bytes
AvmTest::debug_logging: compressed 8916 to 1718 bytes
AvmTest::assert_nullifier_exists: compressed 1964 to 502 bytes
AvmTest::l1_to_l2_msg_exists: compressed 162 to 101 bytes
AvmTest::note_hash_exists: compressed 162 to 102 bytes
AvmTest::emit_unencrypted_log: compressed 47663 to 3297 bytes
AvmTest::send_l2_to_l1_msg: compressed 105 to 79 bytes
AvmTest::get_args_hash: compressed 176 to 105 bytes
AvmTest::get_block_number: compressed 111 to 80 bytes
AvmTest::set_read_storage_single: compressed 1372 to 350 bytes
AvmTest::get_fee_per_l2_gas: compressed 111 to 81 bytes
AvmTest::keccak_hash: compressed 559 to 220 bytes
AvmTest::sha256_hash: compressed 537 to 207 bytes
AvmTest::nullifier_exists: compressed 158 to 99 bytes
AvmTest::new_note_hash: compressed 91 to 74 bytes
AvmTest::set_opcode_u8: compressed 113 to 82 bytes
AvmTest::add_args_return: compressed 140 to 92 bytes
AvmTest::get_storage_address: compressed 111 to 81 bytes
AvmTest::emit_nullifier_and_check: compressed 4100 to 682 bytes
AvmTest::u128_from_integer_overflow: compressed 43568 to 2048 bytes
AvmTest::poseidon2_hash: compressed 55082 to 4986 bytes
AvmTest::set_opcode_u64: compressed 120 to 86 bytes
AvmTest::get_timestamp: compressed 111 to 80 bytes
AvmTest::set_storage_single: compressed 719 to 201 bytes
AvmTest::set_opcode_big_field: compressed 263 to 137 bytes
```

---------

Co-authored-by: dbanks12 <[email protected]>

avm-transpiler/Cargo.toml

@@ -19,4 +19,5 @@ regex = "1.10"
 env_logger = "0.11"
 log = "0.4"
 serde_json = "1.0"
-serde = { version = "1.0.136", features = ["derive"]}
+serde = { version = "1.0.136", features = ["derive"] }
+flate2 = "1.0"

avm-transpiler/src/transpile_contract.rs

@@ -1,9 +1,10 @@
+use std::io::Read;
+
 use base64::Engine;
 use log::info;
 use serde::{Deserialize, Serialize};
 
 use acvm::acir::circuit::Program;
-use noirc_errors::debug_info::DebugInfo;
 use noirc_errors::debug_info::ProgramDebugInfo;
 
 use crate::transpile::{brillig_to_avm, map_brillig_pcs_to_avm_pcs, patch_debug_info_pcs};
@@ -104,6 +105,21 @@ impl From<CompiledAcirContractArtifact> for TranspiledContractArtifact {
                 // Transpile to AVM
                 let avm_bytecode = brillig_to_avm(brillig_bytecode, &brillig_pcs_to_avm_pcs);
 
+                // Gzip AVM bytecode. This has to be removed once we need to do bytecode verification.
+                let mut compressed_avm_bytecode = Vec::new();
+                let mut encoder =
+                    flate2::read::GzEncoder::new(&avm_bytecode[..], flate2::Compression::best());
+                let _ = encoder.read_to_end(&mut compressed_avm_bytecode);
+
+                log::info!(
+                    "{}::{}: compressed {} to {} bytes ({}% reduction)",
+                    contract.name,
+                    function.name,
+                    avm_bytecode.len(),
+                    compressed_avm_bytecode.len(),
+                    100 - (compressed_avm_bytecode.len() * 100 / avm_bytecode.len())
+                );
+
                 // Patch the debug infos with updated PCs
                 let debug_infos = patch_debug_info_pcs(
                     &function.debug_symbols.debug_infos,
@@ -117,7 +133,7 @@ impl From<CompiledAcirContractArtifact> for TranspiledContractArtifact {
                         is_unconstrained: function.is_unconstrained,
                         custom_attributes: function.custom_attributes,
                         abi: function.abi,
-                        bytecode: base64::prelude::BASE64_STANDARD.encode(avm_bytecode),
+                        bytecode: base64::prelude::BASE64_STANDARD.encode(compressed_avm_bytecode),
                         debug_symbols: ProgramDebugInfo { debug_infos },
                     },
                 ));

noir-projects/noir-contracts/scripts/transpile.sh

@@ -2,5 +2,4 @@
 set -eu
 
 TRANSPILER=${TRANSPILER:-../../avm-transpiler/target/release/avm-transpiler}
-ls target/*.json | parallel "$TRANSPILER {} {}"
-
+ls target/*.json | parallel "$TRANSPILER {} {}"

yarn-project/simulator/src/avm/avm_simulator.test.ts

@@ -10,7 +10,7 @@ import { AvmNestedCallsTestContractArtifact, AvmTestContractArtifact } from '@az
 import { jest } from '@jest/globals';
 import { strict as assert } from 'assert';
 
-import { isAvmBytecode } from '../public/transitional_adaptors.js';
+import { isAvmBytecode, markBytecodeAsAvm } from '../public/transitional_adaptors.js';
 import { AvmMachineState } from './avm_machine_state.js';
 import { type MemoryValue, TypeTag, type Uint8 } from './avm_memory_types.js';
 import { AvmSimulator } from './avm_simulator.js';
@@ -39,14 +39,14 @@ describe('AVM simulator: injected bytecode', () => {
     ]);
   });
 
-  it('Should not be recognized as AVM bytecode (magic missing)', () => {
-    expect(!isAvmBytecode(bytecode));
+  it('Should not be recognized as AVM bytecode (magic missing)', async () => {
+    expect(!(await isAvmBytecode(bytecode)));
   });
 
   it('Should execute bytecode that performs basic addition', async () => {
     const context = initContext({ env: initExecutionEnvironment({ calldata }) });
     const { l2GasLeft: initialL2GasLeft } = AvmMachineState.fromState(context.machineState);
-    const results = await new AvmSimulator(context).executeBytecode(bytecode);
+    const results = await new AvmSimulator(context).executeBytecode(markBytecodeAsAvm(bytecode));
 
     expect(results.reverted).toBe(false);
     expect(results.output).toEqual([new Fr(3)]);
@@ -59,7 +59,7 @@ describe('AVM simulator: injected bytecode', () => {
       machineState: initMachineState({ l2GasLeft: 5 }),
     });
 
-    const results = await new AvmSimulator(context).executeBytecode(bytecode);
+    const results = await new AvmSimulator(context).executeBytecode(markBytecodeAsAvm(bytecode));
     expect(results.reverted).toBe(true);
     expect(results.output).toEqual([]);
     expect(results.revertReason?.message).toEqual('Not enough L2GAS gas left');
@@ -91,9 +91,9 @@ describe('AVM simulator: transpiled Noir contracts', () => {
     expect(results.output).toEqual([new Fr(0)]);
   });
 
-  it('Should be recognized as AVM bytecode (magic present)', () => {
+  it('Should be recognized as AVM bytecode (magic present)', async () => {
     const bytecode = getAvmTestContractBytecode('add_args_return');
-    expect(isAvmBytecode(bytecode));
+    expect(await isAvmBytecode(bytecode));
   });
 
   describe('U128 addition and overflows', () => {

yarn-project/simulator/src/avm/avm_simulator.ts

@@ -2,7 +2,7 @@ import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log';
 
 import { strict as assert } from 'assert';
 
-import { isAvmBytecode } from '../public/transitional_adaptors.js';
+import { decompressBytecodeIfCompressed, isAvmBytecode } from '../public/transitional_adaptors.js';
 import type { AvmContext } from './avm_context.js';
 import { AvmContractCallResults } from './avm_message_call_result.js';
 import {
@@ -39,7 +39,6 @@ export class AvmSimulator {
     if (!bytecode) {
       throw new NoBytecodeForContractError(this.context.environment.address);
     }
-    assert(isAvmBytecode(bytecode), "AVM simulator can't execute non-AVM bytecode");
 
     return await this.executeBytecode(bytecode);
   }
@@ -49,7 +48,10 @@ export class AvmSimulator {
    * This method is useful for testing and debugging.
    */
   public async executeBytecode(bytecode: Buffer): Promise<AvmContractCallResults> {
-    return await this.executeInstructions(decodeFromBytecode(bytecode));
+    const decompressedBytecode = await decompressBytecodeIfCompressed(bytecode);
+    assert(isAvmBytecode(decompressedBytecode), "AVM simulator can't execute non-AVM bytecode");
+
+    return await this.executeInstructions(decodeFromBytecode(decompressedBytecode));
   }
 
   /**

yarn-project/simulator/src/public/executor.ts

@@ -11,6 +11,7 @@ import {
 import { createDebugLogger } from '@aztec/foundation/log';
 
 import { spawn } from 'child_process';
+import { assert } from 'console';
 import fs from 'fs/promises';
 import path from 'path';
 
@@ -27,7 +28,12 @@ import { PackedValuesCache } from '../common/packed_values_cache.js';
 import { type CommitmentsDB, type PublicContractsDB, type PublicStateDB } from './db.js';
 import { type PublicExecution, type PublicExecutionResult, checkValidStaticCall } from './execution.js';
 import { PublicExecutionContext } from './public_execution_context.js';
-import { convertAvmResultsToPxResult, createAvmExecutionEnvironment, isAvmBytecode } from './transitional_adaptors.js';
+import {
+  convertAvmResultsToPxResult,
+  createAvmExecutionEnvironment,
+  decompressBytecodeIfCompressed,
+  isAvmBytecode,
+} from './transitional_adaptors.js';
 
 /**
  * Execute a public function and return the execution result.
@@ -46,7 +52,7 @@ export async function executePublicFunction(
     );
   }
 
-  if (isAvmBytecode(bytecode)) {
+  if (await isAvmBytecode(bytecode)) {
     return await executeTopLevelPublicFunctionAvm(context, bytecode);
   } else {
     return await executePublicFunctionAcvm(context, bytecode, nested);
@@ -355,7 +361,10 @@ export class PublicExecutor {
     const proofPath = path.join(artifactsPath, 'proof');
 
     const { args, functionData, contractAddress } = avmExecution;
-    const bytecode = await this.contractsDb.getBytecode(contractAddress, functionData.selector);
+    let bytecode = await this.contractsDb.getBytecode(contractAddress, functionData.selector);
+    assert(!!bytecode, `Bytecode not found for ${contractAddress}:${functionData.selector}`);
+    // This should be removed once we do bytecode validation.
+    bytecode = await decompressBytecodeIfCompressed(bytecode!);
     // Write call data and bytecode to files.
     await fs.writeFile(
       calldataPath,

yarn-project/simulator/src/public/transitional_adaptors.ts

@@ -10,6 +10,9 @@ import {
 } from '@aztec/circuits.js';
 import { Fr } from '@aztec/foundation/fields';
 
+import { promisify } from 'util';
+import { gunzip } from 'zlib';
+
 import { type AvmContext } from '../avm/avm_context.js';
 import { AvmExecutionEnvironment } from '../avm/avm_execution_environment.js';
 import { type AvmContractCallResults } from '../avm/avm_message_call_result.js';
@@ -111,7 +114,19 @@ export function markBytecodeAsAvm(bytecode: Buffer): Buffer {
   return Buffer.concat([bytecode, AVM_MAGIC_SUFFIX]);
 }
 
-export function isAvmBytecode(bytecode: Buffer): boolean {
+// This is just a helper function for the AVM circuit.
+export async function decompressBytecodeIfCompressed(bytecode: Buffer): Promise<Buffer> {
+  try {
+    return await promisify(gunzip)(bytecode);
+  } catch {
+    // If the bytecode is not compressed, the gunzip call will throw an error
+    // In this case, we assume the bytecode is not compressed and continue.
+    return Promise.resolve(bytecode);
+  }
+}
+
+export async function isAvmBytecode(bytecode: Buffer): Promise<boolean> {
+  const decompressedBytecode = await decompressBytecodeIfCompressed(bytecode);
   const magicSize = AVM_MAGIC_SUFFIX.length;
-  return bytecode.subarray(-magicSize).equals(AVM_MAGIC_SUFFIX);
+  return decompressedBytecode.subarray(-magicSize).equals(AVM_MAGIC_SUFFIX);
 }