From 7f8e913d19f568ba17fbda68f3adf4a3d6280b36 Mon Sep 17 00:00:00 2001 From: Blaine Bublitz Date: Tue, 14 Mar 2023 08:36:55 -0700 Subject: [PATCH] feat: Support nix package manager (https://github.com/AztecProtocol/barretenberg/pull/234) * feat: Support nix package manager & add github action to build nix artifacts * Checking gnu flag for buildEnv override * Need to be explicit about gcc11 for aarch64 linux * Use runner.os name for artifacts * Cleanup comments in nixfile * Add concurrency group to workflow * Update the cases when the workflow runs --- .../barretenberg/.github/workflows/nix.yml | 49 +++++++++ circuits/cpp/barretenberg/.gitignore | 3 +- circuits/cpp/barretenberg/barretenberg.nix | 41 ++++++++ circuits/cpp/barretenberg/flake.lock | 43 ++++++++ circuits/cpp/barretenberg/flake.nix | 99 +++++++++++++++++++ 5 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 circuits/cpp/barretenberg/.github/workflows/nix.yml create mode 100644 circuits/cpp/barretenberg/barretenberg.nix create mode 100644 circuits/cpp/barretenberg/flake.lock create mode 100644 circuits/cpp/barretenberg/flake.nix diff --git a/circuits/cpp/barretenberg/.github/workflows/nix.yml b/circuits/cpp/barretenberg/.github/workflows/nix.yml new file mode 100644 index 00000000000..dd76e8301ce --- /dev/null +++ b/circuits/cpp/barretenberg/.github/workflows/nix.yml @@ -0,0 +1,49 @@ +name: Nix builds + +on: + push: + branches: + - phated/** + schedule: + - cron: "0 2 * * *" # run at 2 AM UTC + workflow_dispatch: + +# This will cancel previous runs when a branch or PR is updated +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + nix-build: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + target: [llvm12, wasm32, cross-aarch64] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - uses: cachix/install-nix-action@v20 + with: + nix_path: nixpkgs=channel:nixos-22.11 + github_access_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build barretenberg as ${{ matrix.target }} + run: | + nix build -L .#${{ matrix.target }} + + - name: Prepare artifact + run: | + mkdir artifacts + cp -a ./result/. artifacts + 7z a -ttar -so -an ./artifacts/* | 7z a -si ./libbarretenberg.tar.gz + + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: libbarretenberg-${{ runner.os }}-${{ matrix.target }}.tar.gz + path: ./libbarretenberg.tar.gz diff --git a/circuits/cpp/barretenberg/.gitignore b/circuits/cpp/barretenberg/.gitignore index 600d2d33bad..02b95675015 100644 --- a/circuits/cpp/barretenberg/.gitignore +++ b/circuits/cpp/barretenberg/.gitignore @@ -1 +1,2 @@ -.vscode \ No newline at end of file +.vscode +result diff --git a/circuits/cpp/barretenberg/barretenberg.nix b/circuits/cpp/barretenberg/barretenberg.nix new file mode 100644 index 00000000000..fc00242f4ac --- /dev/null +++ b/circuits/cpp/barretenberg/barretenberg.nix @@ -0,0 +1,41 @@ +{ overrideCC, stdenv, llvmPackages, cmake, ninja, lib, callPackage, binaryen, gcc11 }: +let + targetPlatform = stdenv.targetPlatform; + buildEnv = + if (stdenv.targetPlatform.isGnu && stdenv.targetPlatform.isAarch64) then + # As per https://discourse.nixos.org/t/gcc11stdenv-and-clang/17734/7 since it seems that aarch64-linux uses + # gcc9 instead of gcc11 for the C++ stdlib, while all other targets we support provide the correct libstdc++ + overrideCC llvmPackages.stdenv (llvmPackages.clang.override { gccForLibs = gcc11.cc; }) + else + llvmPackages.stdenv; + optionals = lib.lists.optionals; + toolchain_file = ./cpp/cmake/toolchains/${targetPlatform.system}.cmake; +in +buildEnv.mkDerivation +{ + pname = "libbarretenberg"; + version = "0.1.0"; + + src = ./cpp; + + nativeBuildInputs = [ cmake ninja ] + ++ optionals targetPlatform.isWasm [ binaryen ]; + + buildInputs = [ ] + ++ optionals (targetPlatform.isDarwin || targetPlatform.isLinux) [ + llvmPackages.openmp + ]; + + cmakeFlags = [ + "-DTESTING=OFF" + "-DBENCHMARKS=OFF" + "-DCMAKE_TOOLCHAIN_FILE=${toolchain_file}" + ] + ++ optionals (targetPlatform.isDarwin || targetPlatform.isLinux) + [ "-DCMAKE_BUILD_TYPE=RelWithAssert" ]; + + NIX_CFLAGS_COMPILE = + optionals targetPlatform.isDarwin [ " -fno-aligned-allocation" ]; + + enableParallelBuilding = true; +} diff --git a/circuits/cpp/barretenberg/flake.lock b/circuits/cpp/barretenberg/flake.lock new file mode 100644 index 00000000000..4128dfe7e35 --- /dev/null +++ b/circuits/cpp/barretenberg/flake.lock @@ -0,0 +1,43 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1676283394, + "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1678230755, + "narHash": "sha256-SFAXgNjNTXzcAideXcP0takfUGVft/VR5CACmYHg+Fc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a7cc81913bb3cd1ef05ed0ece048b773e1839e51", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/circuits/cpp/barretenberg/flake.nix b/circuits/cpp/barretenberg/flake.nix new file mode 100644 index 00000000000..1d4dd4d7533 --- /dev/null +++ b/circuits/cpp/barretenberg/flake.nix @@ -0,0 +1,99 @@ +{ + description = + "Barretenberg: C++ cryptographic library, BN254 elliptic curve library, and PLONK SNARK prover"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + let + barretenbergOverlay = self: super: { + # It seems that llvmPackages_11 can't build WASI, so default to llvmPackages_12 + barretenberg = super.callPackage ./barretenberg.nix { + llvmPackages = self.llvmPackages_12; + }; + }; + in + flake-utils.lib.eachDefaultSystem + (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ barretenbergOverlay ]; + }; + + optional = pkgs.lib.lists.optional; + + crossTargets = builtins.listToAttrs + ( + [ ] ++ optional (pkgs.hostPlatform.isx86_64 && pkgs.hostPlatform.isLinux) { + name = "cross-aarch64"; + value = pkgs.pkgsCross.aarch64-multiplatform-musl.pkgsLLVM.barretenberg; + } ++ optional (pkgs.hostPlatform.isx86_64 && pkgs.hostPlatform.isDarwin) { + name = "cross-aarch64"; + value = pkgs.pkgsCross.aarch64-darwin.barretenberg.override { + # llvmPackages_12 seems to fail when we try to cross-compile but llvmPackages_11 works + llvmPackages = pkgs.llvmPackages_11; + }; + } + ); + + shellDefaults = { + nativeBuildInputs = [ + pkgs.starship + pkgs.llvmPackages_12.llvm + ]; + + shellHook = '' + eval "$(starship init bash)" + ''; + }; + in + rec { + packages = { + llvm11 = pkgs.barretenberg.override { + llvmPackages = pkgs.llvmPackages_11; + }; + llvm12 = pkgs.barretenberg; + llvm13 = pkgs.barretenberg.override { + llvmPackages = pkgs.llvmPackages_13; + }; + llvm14 = pkgs.barretenberg.override { + llvmPackages = pkgs.llvmPackages_14; + }; + wasm32 = pkgs.pkgsCross.wasi32.barretenberg; + + # Defaulting to llvm12 so we can ensure consistent shells + default = packages.llvm12; + } // crossTargets; + + # Provide legacyPackages with our overlay so we can run + # > `nix build .#pkgsCross.aarch64-multiplatform.barretenberg` + # Ref https://discourse.nixos.org/t/how-do-i-cross-compile-a-flake/12062/12 + legacyPackages = import nixpkgs { + inherit system; + overlays = [ barretenbergOverlay ]; + crossOverlays = [ barretenbergOverlay ]; + }; + + devShells = { + default = pkgs.mkShell.override { stdenv = packages.default.stdenv; } + ({ + inputsFrom = + [ packages.default ]; + } // shellDefaults); + + wasm32 = pkgs.mkShell.override + { + stdenv = packages.wasm32.stdenv; + } + ({ + inputsFrom = [ packages.wasm32 ]; + } // shellDefaults); + }; + }) // { + overlays.default = barretenbergOverlay; + }; +}