feat!: Unify ABIs between nargo and yarn-project (#3989)

Changes the output of `nargo` and `noir-wasm` to generate a single
artifact that includes debug information. Compiled contracts now have a
`file_map` at the root of the artifact, and a `debug_symbols` within
each function. For consistency, compiled programs also have `file_map`
and `debug_symbols`. Debug symbols are compressed and base64-encoded, as
they were by the former `noir-compiler` postprocessing. The `debug` json
artifact is no longer emitted.

Removes all code related to generating `yarn-project`-specific ABIs from
noir compiler. Instead, `types` now exposes a `loadContractArtifact`
that renames fields as needed (and use camel instead of snake case), and
is required when loading a noir contract artifact. Autogenerated types
run this automatically.

Since we are no longer post-processing artifacts, the `noir-contracts`
project shape is changed. JSON files live in the `target` folder where
nargo outputs them (this is not configurable), and are loaded by the
autogenerated typescript interfaces from there.

As part of the refactor, moves functions for getting functions debug
info out of `acir-simulator` and into `types`.

**Breaking change**: This removes the need to run `codegen` for using a
compiled contract. However, when creating a new contract object from
aztec.js, a dev needs to call `loadContractArtifact`.

**Future changes**: Type information for a compilation artifact is now
spread all over the place, and duplicated in more than one place. There
are types defined within rust code in Noir as
`[wasm_bindgen(typescript_custom_section)]`, others defined within
typescript code in the `noir_wasm` package, others in `foundation`, and
others in `types`. We should unify it in a single place.

Fixes AztecProtocol/aztec-packages#3812

Supersedes #3906

Noir subrepo has been pushed to
#4035 to run the Noir CI

Cargo.toml

@@ -123,6 +123,7 @@ num-traits = "0.2"
 similar-asserts = "1.5.0"
 tempfile = "3.6.0"
 jsonrpc = { version = "0.16.0", features = ["minreq_http"] }
+flate2 = "1.0.24"
 
 tracing = "0.1.40"
 tracing-web = "0.1.3"

acvm-repo/acir/Cargo.toml

@@ -17,7 +17,7 @@ acir_field.workspace = true
 brillig.workspace = true
 serde.workspace = true
 thiserror.workspace = true
-flate2 = "1.0.24"
+flate2.workspace = true
 bincode.workspace = true
 base64.workspace = true
 

compiler/noirc_errors/Cargo.toml

@@ -15,4 +15,7 @@ fm.workspace = true
 chumsky.workspace = true
 serde.workspace = true
 serde_with = "3.2.0"
-tracing.workspace = true
+tracing.workspace = true
+flate2.workspace = true
+serde_json.workspace = true
+base64.workspace = true

compiler/noirc_errors/src/debug_info.rs

@@ -1,14 +1,24 @@
 use acvm::acir::circuit::OpcodeLocation;
 use acvm::compiler::AcirTransformationMap;
 
+use base64::Engine;
+use flate2::read::DeflateDecoder;
+use flate2::write::DeflateEncoder;
+use flate2::Compression;
+use serde::Deserializer;
+use serde::Serializer;
 use serde_with::serde_as;
 use serde_with::DisplayFromStr;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
+use std::io::Read;
+use std::io::Write;
 use std::mem;
 
 use crate::Location;
-use serde::{Deserialize, Serialize};
+use serde::{
+    de::Error as DeserializationError, ser::Error as SerializationError, Deserialize, Serialize,
+};
 
 #[serde_as]
 #[derive(Default, Debug, Clone, Deserialize, Serialize)]
@@ -86,4 +96,40 @@ impl DebugInfo {
 
         counted_opcodes
     }
+
+    pub fn serialize_compressed_base64_json<S>(
+        debug_info: &DebugInfo,
+        s: S,
+    ) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        let json_str = serde_json::to_string(debug_info).map_err(S::Error::custom)?;
+
+        let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default());
+        encoder.write_all(json_str.as_bytes()).map_err(S::Error::custom)?;
+        let compressed_data = encoder.finish().map_err(S::Error::custom)?;
+
+        let encoded_b64 = base64::prelude::BASE64_STANDARD.encode(compressed_data);
+        s.serialize_str(&encoded_b64)
+    }
+
+    pub fn deserialize_compressed_base64_json<'de, D>(
+        deserializer: D,
+    ) -> Result<DebugInfo, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let encoded_b64: String = Deserialize::deserialize(deserializer)?;
+
+        let compressed_data =
+            base64::prelude::BASE64_STANDARD.decode(encoded_b64).map_err(D::Error::custom)?;
+
+        let mut decoder = DeflateDecoder::new(&compressed_data[..]);
+        let mut decompressed_data = Vec::new();
+        decoder.read_to_end(&mut decompressed_data).map_err(D::Error::custom)?;
+
+        let json_str = String::from_utf8(decompressed_data).map_err(D::Error::custom)?;
+        serde_json::from_str(&json_str).map_err(D::Error::custom)
+    }
 }

compiler/wasm/Cargo.toml

@@ -18,6 +18,7 @@ nargo.workspace = true
 noirc_driver.workspace = true
 noirc_frontend.workspace = true
 noirc_errors.workspace = true
+noirc_evaluator.workspace = true
 wasm-bindgen.workspace = true
 serde.workspace = true
 js-sys.workspace = true

compiler/wasm/package.json

@@ -34,6 +34,7 @@
     "test": "yarn test:build_fixtures && yarn test:node && yarn test:browser",
     "test:build_fixtures": "./scripts/build-fixtures.sh",
     "test:browser": "web-test-runner",
+    "test:browser:docker": "docker run --rm -v $(cd ../.. && pwd):/usr/src/noir -w /usr/src/noir/compiler/wasm mcr.microsoft.com/playwright:v1.40.0-jammy yarn test:browser",
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
     "clean": "rm -rf ./build ./target ./dist public/fixtures/simple/target public/fixtures/with-deps/target",
     "nightly:version": "jq --arg new_version \"-$(git rev-parse --short HEAD)$1\" '.version = .version + $new_version' package.json > package-tmp.json && mv package-tmp.json package.json",
@@ -49,6 +50,7 @@
     "@types/mocha": "^10.0.6",
     "@types/mocha-each": "^2",
     "@types/node": "^20.10.5",
+    "@types/pako": "^2",
     "@types/path-browserify": "^1",
     "@types/readable-stream": "^4",
     "@types/sinon": "^17",
@@ -76,5 +78,8 @@
     "url": "^0.11.3",
     "webpack": "^5.49.0",
     "webpack-cli": "^4.7.2"
+  },
+  "dependencies": {
+    "pako": "^2.1.0"
   }
 }

compiler/wasm/src/compile.rs

@@ -3,14 +3,14 @@ use gloo_utils::format::JsValueSerdeExt;
 use js_sys::{JsString, Object};
 use nargo::artifacts::{
     contract::{ContractArtifact, ContractFunctionArtifact},
-    debug::DebugArtifact,
     program::ProgramArtifact,
 };
 use noirc_driver::{
     add_dep, compile_contract, compile_main, file_manager_with_stdlib, prepare_crate,
     prepare_dependency, CompileOptions, CompiledContract, CompiledProgram,
     NOIR_ARTIFACT_VERSION_STRING,
 };
+use noirc_evaluator::errors::SsaReport;
 use noirc_frontend::{
     graph::{CrateId, CrateName},
     hir::Context,
@@ -28,35 +28,30 @@ export type DependencyGraph = {
     library_dependencies: Readonly<Record<string, readonly string[]>>;
 }
 
-export type CompiledContract = {
+export type ContractArtifact = {
     noir_version: string;
     name: string;
     functions: Array<any>;
     events: Array<any>;
+    file_map: Record<number, any>;
 };
 
-export type CompiledProgram = {
+export type ProgramArtifact = {
     noir_version: string;
+    hash: number;
     abi: any;
     bytecode: string;
+    debug_symbols: any;
+    file_map: Record<number, any>;
 }
 
-export type DebugArtifact = {
-    debug_symbols: Array<any>;
-    file_map: Record<number, any>;
-    warnings: Array<any>;
-};
+type WarningsCompileResult = { warnings: Array<any>; };
 
-export type CompileResult = (
-    | {
-        contract: CompiledContract;
-        debug: DebugArtifact;
-    }
-    | {
-        program: CompiledProgram;
-        debug: DebugArtifact;
-    }
-);
+export type ContractCompileResult = { contract: CompiledContract; } & WarningsCompileResult;
+
+export type ProgramCompileResult = { program: CompiledProgram; } & WarningsCompileResult;
+
+export type CompileResult = ContractCompileResult | ProgramCompileResult;
 "#;
 
 #[wasm_bindgen]
@@ -76,38 +71,36 @@ extern "C" {
 impl JsCompileResult {
     const CONTRACT_PROP: &'static str = "contract";
     const PROGRAM_PROP: &'static str = "program";
-    const DEBUG_PROP: &'static str = "debug";
+    const WARNINGS_PROP: &'static str = "warnings";
 
     pub fn new(resp: CompileResult) -> JsCompileResult {
         let obj = JsCompileResult::constructor();
         match resp {
-            CompileResult::Contract { contract, debug } => {
+            CompileResult::Contract { contract, warnings } => {
                 js_sys::Reflect::set(
                     &obj,
                     &JsString::from(JsCompileResult::CONTRACT_PROP),
                     &<JsValue as JsValueSerdeExt>::from_serde(&contract).unwrap(),
                 )
                 .unwrap();
-
                 js_sys::Reflect::set(
                     &obj,
-                    &JsString::from(JsCompileResult::DEBUG_PROP),
-                    &<JsValue as JsValueSerdeExt>::from_serde(&debug).unwrap(),
+                    &JsString::from(JsCompileResult::WARNINGS_PROP),
+                    &<JsValue as JsValueSerdeExt>::from_serde(&warnings).unwrap(),
                 )
                 .unwrap();
             }
-            CompileResult::Program { program, debug } => {
+            CompileResult::Program { program, warnings } => {
                 js_sys::Reflect::set(
                     &obj,
                     &JsString::from(JsCompileResult::PROGRAM_PROP),
                     &<JsValue as JsValueSerdeExt>::from_serde(&program).unwrap(),
                 )
                 .unwrap();
-
                 js_sys::Reflect::set(
                     &obj,
-                    &JsString::from(JsCompileResult::DEBUG_PROP),
-                    &<JsValue as JsValueSerdeExt>::from_serde(&debug).unwrap(),
+                    &JsString::from(JsCompileResult::WARNINGS_PROP),
+                    &<JsValue as JsValueSerdeExt>::from_serde(&warnings).unwrap(),
                 )
                 .unwrap();
             }
@@ -148,8 +141,8 @@ impl PathToFileSourceMap {
 }
 
 pub enum CompileResult {
-    Contract { contract: ContractArtifact, debug: DebugArtifact },
-    Program { program: ProgramArtifact, debug: DebugArtifact },
+    Contract { contract: ContractArtifact, warnings: Vec<SsaReport> },
+    Program { program: ProgramArtifact, warnings: Vec<SsaReport> },
 }
 
 #[wasm_bindgen]
@@ -273,31 +266,22 @@ fn add_noir_lib(context: &mut Context, library_name: &CrateName) -> CrateId {
 }
 
 pub(crate) fn generate_program_artifact(program: CompiledProgram) -> CompileResult {
-    let debug_artifact = DebugArtifact {
-        debug_symbols: vec![program.debug.clone()],
-        file_map: program.file_map.clone(),
-        warnings: program.warnings.clone(),
-    };
-
-    CompileResult::Program { program: program.into(), debug: debug_artifact }
+    let warnings = program.warnings.clone();
+    CompileResult::Program { program: program.into(), warnings }
 }
 
 pub(crate) fn generate_contract_artifact(contract: CompiledContract) -> CompileResult {
-    let debug_artifact = DebugArtifact {
-        debug_symbols: contract.functions.iter().map(|function| function.debug.clone()).collect(),
-        file_map: contract.file_map,
-        warnings: contract.warnings,
-    };
     let functions = contract.functions.into_iter().map(ContractFunctionArtifact::from).collect();
 
     let contract_artifact = ContractArtifact {
         noir_version: String::from(NOIR_ARTIFACT_VERSION_STRING),
         name: contract.name,
         functions,
         events: contract.events,
+        file_map: contract.file_map,
     };
 
-    CompileResult::Contract { contract: contract_artifact, debug: debug_artifact }
+    CompileResult::Contract { contract: contract_artifact, warnings: contract.warnings }
 }
 
 #[cfg(test)]

compiler/wasm/src/index.cts

@@ -3,6 +3,7 @@ import { createNodejsFileManager } from './noir/file-manager/nodejs-file-manager
 import { NoirWasmCompiler } from './noir/noir-wasm-compiler';
 import { LogData, LogFn } from './utils';
 import { CompilationResult } from './types/noir_artifact';
+import { inflateDebugSymbols } from './noir/debug';
 
 async function compile(
   fileManager: FileManager,
@@ -46,4 +47,4 @@ async function compile(
 
 const createFileManager = createNodejsFileManager;
 
-export { compile, createFileManager, CompilationResult };
+export { compile, createFileManager, inflateDebugSymbols, CompilationResult };

compiler/wasm/src/index.mts

@@ -3,6 +3,7 @@ import { createNodejsFileManager } from './noir/file-manager/nodejs-file-manager
 import { NoirWasmCompiler } from './noir/noir-wasm-compiler';
 import { LogData, LogFn } from './utils';
 import { CompilationResult } from './types/noir_artifact';
+import { inflateDebugSymbols } from './noir/debug';
 
 async function compile(
   fileManager: FileManager,
@@ -48,4 +49,4 @@ async function compile(
 
 const createFileManager = createNodejsFileManager;
 
-export { compile, createFileManager, CompilationResult };
+export { compile, createFileManager, inflateDebugSymbols, CompilationResult };

compiler/wasm/src/noir/debug.ts

@@ -0,0 +1,6 @@
+import { inflate } from 'pako';
+
+/** Decompresses and decodes the debug symbols */
+export function inflateDebugSymbols(debugSymbols: string) {
+  return JSON.parse(inflate(Buffer.from(debugSymbols, 'base64'), { to: 'string', raw: true }));
+}