feat: Add dependency resolver for noir_wasm and implement FileManager for consistency with native interface

.github/workflows/test-js-packages.yml

@@ -47,26 +47,37 @@ jobs:
           retention-days: 3
 
   build-noir-wasm:
+    needs: [build-noirc-abi]
     runs-on: ubuntu-latest
     timeout-minutes: 30
 
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
 
-      - name: Setup Nix
-        uses: ./.github/actions/nix
+      - name: Setup toolchain
+        uses: dtolnay/[email protected]
+
+      - uses: Swatinem/rust-cache@v2
         with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          nix-cache-name: "noir"
-          cachix-auth-token: ${{ secrets.CACHIXAUTHTOKEN }}
+          key: x86_64-unknown-linux-gnu
+          cache-on-failure: true
+          save-if: ${{ github.event_name != 'merge_group' }}
 
-      - name: Build wasm package
-        run: |
-          nix build -L .#noir_wasm
+      - name: Download noirc_abi_wasm package artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: noirc_abi_wasm
+          path: ./tooling/noirc_abi_wasm
+
+      - name: Install Yarn dependencies
+        uses: ./.github/actions/setup
+
+      - name: Build noir_wasm
+        run: yarn workspace @noir-lang/noir_wasm build
 
       - name: Dereference symlink
-        run: echo "UPLOAD_PATH=$(readlink -f ./result/noir_wasm)" >> $GITHUB_ENV
+        run: echo "UPLOAD_PATH=$(readlink -f ./compiler/wasm)" >> $GITHUB_ENV
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3
@@ -75,7 +86,6 @@ jobs:
           path: ${{ env.UPLOAD_PATH }}
           retention-days: 3
 
-
   build-acvm-js:
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -289,35 +299,28 @@ jobs:
           name: noir_wasm
           path: ./compiler/wasm
 
+      - name: Install Yarn dependencies
+        uses: ./.github/actions/setup
+
       - name: Download nargo binary
         uses: actions/download-artifact@v3
         with:
           name: nargo
           path: ./nargo
 
-      - name: Compile fixtures with Nargo CLI
-        working-directory: ./compiler/wasm/fixtures
+      - name: Set nargo on PATH
         run: |
-          nargo_binary=${{ github.workspace }}/nargo/nargo
+          nargo_binary="${{ github.workspace }}/nargo/nargo"
           chmod +x $nargo_binary
-          for dir in $(ls -d */); do
-            pushd $dir/noir-script
-            $nargo_binary compile
-            popd
-          done
+          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
+          export PATH="$PATH:$(dirname $nargo_binary)"
 
-      - name: Install Yarn dependencies
-        uses: ./.github/actions/setup
-
-      - name: Install Playwright
-        uses: ./.github/actions/install-playwright
+      - name: Build fixtures
+        run: yarn workspace @noir-lang/noir_wasm test:build_fixtures
 
       - name: Run node tests
         run: yarn workspace @noir-lang/noir_wasm test:node
 
-      - name: Run browser tests
-        run: yarn workspace @noir-lang/noir_wasm test:browser
-
   test-noir-codegen:
     needs: [build-acvm-js, build-noirc-abi]
     name: noir_codegen

.vscode/settings.json

@@ -18,5 +18,12 @@
     },
     "[javascript]": {
         "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "jestrunner.projectPath": "./",
+    "jestrunner.debugOptions": {
+        "runtimeArgs": ["--experimental-vm-modules"],
+        "env": {
+            "NODE_NO_WARNINGS": 1
+        }
     }
 }

compiler/integration-tests/circuits/deps_testing/Nargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "test"
+authors = [""]
+compiler_version = ">=0.18.0"
+type = "contract"
+
+[dependencies]
+test = { path = "../lib_with_module" }

compiler/integration-tests/circuits/deps_testing/src/main.nr

@@ -0,0 +1,12 @@
+contract TestContract {
+    use dep::test::module::foo;
+
+    fn constructor(param: Field, pub_param: pub Field) -> pub [Field; 2] {
+        [foo::bar(param), param + pub_param]
+    }
+
+    open fn openFunction() -> pub Field {
+        42
+    }
+
+}

compiler/integration-tests/circuits/lib_with_module/Nargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "lib_c"
+authors = [""]
+compiler_version = ">=0.18.0"
+type = "lib"
+
+[dependencies]

compiler/integration-tests/circuits/lib_with_module/src/lib.nr

@@ -0,0 +1 @@
+mod module;

compiler/integration-tests/circuits/lib_with_module/src/module.nr

@@ -0,0 +1 @@
+mod foo;

compiler/integration-tests/circuits/lib_with_module/src/module/foo.nr

@@ -0,0 +1,3 @@
+pub fn bar(param: Field) -> Field {
+  dep::std::hash::pedersen_hash([param])
+}

compiler/integration-tests/scripts/codegen-verifiers.sh

@@ -21,6 +21,11 @@ contracts_dir=$self_path/../contracts
 rm -rf $contracts_dir
 mkdir $contracts_dir
 
+# Build dependency testing contract
+# Run codegen-verifier for recursion
+dep_testing_dir=$repo_root/compiler/integration-tests/circuits/deps_testing
+nargo --program-dir $dep_testing_dir compile
+
 cp $mul_dir/contract/1_mul/plonk_vk.sol $contracts_dir/1_mul.sol
 cp $assert_statement_dir/contract/assert_statement/plonk_vk.sol $contracts_dir/assert_statement.sol
 cp $recursion_dir/contract/recursion/plonk_vk.sol $contracts_dir/recursion.sol

compiler/integration-tests/test/browser/compile_prove_verify.test.ts

@@ -1,22 +1,13 @@
 import { expect } from '@esm-bundle/chai';
 import * as TOML from 'smol-toml';
 
-import newCompiler, {
-  CompiledProgram,
-  PathToFileSourceMap,
-  compile,
-  init_log_level as compilerLogLevel,
-} from '@noir-lang/noir_wasm';
+import { NoirCompiledCircuit, compile, createFileManager } from '@noir-lang/noir_wasm';
 import { Noir } from '@noir-lang/noir_js';
 import { InputMap } from '@noir-lang/noirc_abi';
 import { BarretenbergBackend } from '@noir-lang/backend_barretenberg';
 
 import { getFile } from './utils.js';
 
-await newCompiler();
-
-compilerLogLevel('INFO');
-
 const test_cases = [
   {
     case: 'test_programs/execution_success/1_mul',
@@ -32,12 +23,11 @@ const suite = Mocha.Suite.create(mocha.suite, 'Noir end to end test');
 
 suite.timeout(60 * 20e3); //20mins
 
-function getCircuit(noirSource: string): CompiledProgram {
-  const sourceMap = new PathToFileSourceMap();
-  sourceMap.add_source_code('main.nr', noirSource);
-
-  // We're ignoring this in the resolver but pass in something sensible.
-  const result = compile('main.nr', undefined, undefined, sourceMap);
+async function getCircuit(projectPath: string): Promise<NoirCompiledCircuit> {
+  const fm = createFileManager('/');
+  await fm.writeFile('./src/main.nr', await getFile(`${projectPath}/src/main.nr`));
+  await fm.writeFile('./Nargo.toml', await getFile(`${projectPath}/Nargo.toml`));
+  const [result] = await compile(fm);
   if (!('program' in result)) {
     throw new Error('Compilation failed');
   }
@@ -51,11 +41,9 @@ test_cases.forEach((testInfo) => {
     const base_relative_path = '../../../../..';
     const test_case = testInfo.case;
 
-    const noir_source = await getFile(`${base_relative_path}/${test_case}/src/main.nr`);
-
-    let noir_program: CompiledProgram;
+    let noir_program: NoirCompiledCircuit;
     try {
-      noir_program = getCircuit(noir_source);
+      noir_program = await getCircuit(`${base_relative_path}/${test_case}`);
 
       expect(noir_program, 'Compile output ').to.be.an('object');
     } catch (e) {
@@ -66,7 +54,7 @@ test_cases.forEach((testInfo) => {
     const backend = new BarretenbergBackend(noir_program);
     const program = new Noir(noir_program, backend);
 
-    const prover_toml = await getFile(`${base_relative_path}/${test_case}/Prover.toml`);
+    const prover_toml = await new Response(await getFile(`${base_relative_path}/${test_case}/Prover.toml`)).text();
     const inputs: InputMap = TOML.parse(prover_toml) as InputMap;
 
     // JS Proving

compiler/integration-tests/test/browser/compile_with_deps.test.ts

@@ -0,0 +1,40 @@
+import { compile, createFileManager } from '@noir-lang/noir_wasm';
+import { getPaths } from '../shared';
+import { expect } from '@esm-bundle/chai';
+
+const paths = getPaths('.');
+
+async function getFile(path: string) {
+  const basePath = new URL('./../../', import.meta.url).toString().replace(/\/$/g, '');
+  const url = `${basePath}${path.replace('.', '')}`;
+  const response = await fetch(url);
+  return response;
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+async function getPrecompiledSource(path: string): Promise<any> {
+  const response = await getFile(path);
+  const compiledData = await response.text();
+  return JSON.parse(compiledData);
+}
+
+describe('noir-compiler', () => {
+  it('both nargo and noir_wasm should compile identically', async () => {
+    const { contractExpectedArtifact } = paths;
+    const fm = createFileManager('/');
+    const files = Object.values(paths).filter((fileOrDir) => /^\.?\/.*\..*$/.test(fileOrDir));
+    for (const path of files) {
+      await fm.writeFile(path, (await getFile(path)).body as ReadableStream<Uint8Array>);
+    }
+    const nargoArtifact = await getPrecompiledSource(contractExpectedArtifact);
+    nargoArtifact.functions.sort((a, b) => a.name.localeCompare(b.name));
+    const [noirWasmArtifact] = await compile(fm, '/circuits/deps_testing');
+    if (!('contract' in noirWasmArtifact)) {
+      throw new Error('Compilation failed');
+    }
+    const noirWasmContract = noirWasmArtifact.contract;
+    expect(noirWasmContract).not.to.be.undefined;
+    noirWasmContract.functions.sort((a, b) => a.name.localeCompare(b.name));
+    expect(nargoArtifact).to.deep.eq(noirWasmContract);
+  });
+});

compiler/integration-tests/test/browser/recursion.test.ts

@@ -2,59 +2,46 @@
 import { expect } from '@esm-bundle/chai';
 import { TEST_LOG_LEVEL } from '../environment.js';
 import { Logger } from 'tslog';
-import newCompiler, {
-  CompiledProgram,
-  PathToFileSourceMap,
-  compile,
-  init_log_level as compilerLogLevel,
-} from '@noir-lang/noir_wasm';
 import { acvm, abi, Noir } from '@noir-lang/noir_js';
 
 import * as TOML from 'smol-toml';
 import { BarretenbergBackend } from '@noir-lang/backend_barretenberg';
 import { getFile } from './utils.js';
 import { Field, InputMap } from '@noir-lang/noirc_abi';
+import { NoirCompiledCircuit, createFileManager, compile } from '@noir-lang/noir_wasm';
 
 const logger = new Logger({ name: 'test', minLevel: TEST_LOG_LEVEL });
 
 const { default: initACVM } = acvm;
 const { default: newABICoder } = abi;
 
-await newCompiler();
 await newABICoder();
 await initACVM();
 
-compilerLogLevel('INFO');
-
 const base_relative_path = '../../../../..';
 const circuit_main = 'test_programs/execution_success/assert_statement';
 const circuit_recursion = 'compiler/integration-tests/circuits/recursion';
 
-function getCircuit(noirSource: string): CompiledProgram {
-  const sourceMap = new PathToFileSourceMap();
-  sourceMap.add_source_code('main.nr', noirSource);
-  const result = compile('main.nr', undefined, undefined, sourceMap);
+async function getCircuit(projectPath: string): Promise<NoirCompiledCircuit> {
+  const fm = createFileManager('/');
+  await fm.writeFile('./src/main.nr', await getFile(`${projectPath}/src/main.nr`));
+  await fm.writeFile('./Nargo.toml', await getFile(`${projectPath}/Nargo.toml`));
+  const [result] = await compile(fm);
   if (!('program' in result)) {
     throw new Error('Compilation failed');
   }
-
   return result.program;
 }
 
 describe('It compiles noir program code, receiving circuit bytes and abi object.', () => {
-  let circuit_main_source;
   let circuit_main_toml;
-  let circuit_recursion_source;
 
   before(async () => {
-    circuit_main_source = await getFile(`${base_relative_path}/${circuit_main}/src/main.nr`);
-    circuit_main_toml = await getFile(`${base_relative_path}/${circuit_main}/Prover.toml`);
-
-    circuit_recursion_source = await getFile(`${base_relative_path}/${circuit_recursion}/src/main.nr`);
+    circuit_main_toml = await new Response(await getFile(`${base_relative_path}/${circuit_main}/Prover.toml`)).text();
   });
 
   it('Should generate valid inner proof for correct input, then verify proof within a proof', async () => {
-    const main_program = getCircuit(circuit_main_source);
+    const main_program = await getCircuit(`${base_relative_path}/${circuit_main}`);
     const main_inputs: InputMap = TOML.parse(circuit_main_toml) as InputMap;
 
     const main_backend = new BarretenbergBackend(main_program);
@@ -84,7 +71,7 @@ describe('It compiles noir program code, receiving circuit bytes and abi object.
 
     logger.debug('recursion_inputs', recursion_inputs);
 
-    const recursion_program = await getCircuit(circuit_recursion_source);
+    const recursion_program = await getCircuit(`${base_relative_path}/${circuit_recursion}`);
 
     const recursion_backend = new BarretenbergBackend(recursion_program);
 

compiler/integration-tests/test/browser/utils.ts

@@ -1,8 +1,8 @@
-export async function getFile(file_path: string): Promise<string> {
+export async function getFile(file_path: string): Promise<ReadableStream<Uint8Array>> {
   const file_url = new URL(file_path, import.meta.url);
   const response = await fetch(file_url);
 
   if (!response.ok) throw new Error('Network response was not OK');
 
-  return await response.text();
+  return response.body as ReadableStream<Uint8Array>;
 }

compiler/integration-tests/test/node/compile_with_deps.test.ts

@@ -0,0 +1,26 @@
+import { join, resolve } from 'path';
+import { getPaths } from '../shared';
+
+import { expect } from 'chai';
+import { readFile } from 'fs/promises';
+import { compile, createFileManager } from '@noir-lang/noir_wasm';
+import { NoirCompiledContract } from '@noir-lang/noir_wasm/dist/types/types/noir_artifact';
+
+const basePath = resolve(join(__dirname, '../../'));
+const { contractProjectPath, contractExpectedArtifact } = getPaths(basePath);
+
+describe('noir-compiler', () => {
+  it('both nargo and noir_wasm should compile identically', async () => {
+    const fm = createFileManager(contractProjectPath);
+    const nargoArtifact = JSON.parse((await readFile(contractExpectedArtifact)).toString()) as NoirCompiledContract;
+    nargoArtifact.functions.sort((a, b) => a.name.localeCompare(b.name));
+    const [noirWasmArtifact] = await compile(fm);
+    if (!('contract' in noirWasmArtifact)) {
+      throw new Error('Compilation failed');
+    }
+    const noirWasmContract = noirWasmArtifact.contract;
+    expect(noirWasmContract).not.to.be.undefined;
+    noirWasmContract.functions.sort((a, b) => a.name.localeCompare(b.name));
+    expect(nargoArtifact).to.deep.eq(noirWasmContract);
+  });
+});