From 968d3cbfe8db4fa5ada6eb17283716eeb382c85f Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Fri, 17 Mar 2023 11:59:58 +0000 Subject: [PATCH] CI attempt 2 (#156) * CI attempt 2 * Revise cardano-crypto-class * Don't check out builder any more * Small fixes for the builder * later compiler * Try this * More new compiler * Try multiline again * Add some extra config for IFD * Tweaks * Add nixbuild.net as a substituter * Major flake restructuring * Just plutus-core for now * Doc * Comments * Refactor * Bump hackage.nix and drop vector workaround * Fix action version --- .github/workflows/CI.yml | 63 ++++++-- README.md | 28 +++- .../cardano-crypto-class/2.0.0.1/meta.toml | 4 + .../2.0.0.1/revisions/1.cabal | 139 ++++++++++++++++++ builder/default.nix | 56 ------- flake.lock | 6 +- flake.nix | 68 +++++++-- nix/builder.nix | 58 ++++++++ nix/chap-meta.nix | 29 ++++ {builder => nix}/empty/almost/.gitkeep | 0 10 files changed, 362 insertions(+), 89 deletions(-) create mode 100644 _sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal delete mode 100644 builder/default.nix create mode 100644 nix/builder.nix create mode 100644 nix/chap-meta.nix rename {builder => nix}/empty/almost/.gitkeep (100%) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8ed33893a..4098ee20d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,25 +20,23 @@ jobs: fetch-depth: 0 # the check script below needs the whole history - name: Setup Nix - uses: cachix/install-nix-action@v17 + uses: cachix/install-nix-action@v20 with: nix_path: nixpkgs=channel:nixos-unstable - extra_nix_config: | - accept-flake-config = true + extra_nix_config: accept-flake-config = true - name: Run checks run: nix develop -c ./scripts/check.sh - build: + build-repo: runs-on: ubuntu-latest steps: - name: Setup Nix - uses: cachix/install-nix-action@v17 + uses: cachix/install-nix-action@v20 with: nix_path: nixpkgs=channel:nixos-unstable - extra_nix_config: | - accept-flake-config = true + extra_nix_config: accept-flake-config = true - name: Checkout main uses: actions/checkout@v3 @@ -64,8 +62,10 @@ jobs: fi - name: Build repository (main) + # We don't need the metadata here, since we're just + # using this to compare the generated index run: | - nix develop -c foliage build -j 0 + nix develop -c foliage build -j 0 mv _repo _repo-main - name: Checkout tip commit @@ -84,7 +84,7 @@ jobs: # Do this before the check, useful to have the artifact in case the # check fails! - - name: Upload artifact + - name: Upload built repository uses: actions/upload-artifact@v3 with: name: built-repo @@ -100,11 +100,52 @@ jobs: echo "then you may need to update the timestamps in your new packages to be newer than those in main." ./scripts/check-archive-extension.sh _repo-main/01-index.tar _repo/01-index.tar + build-packages: + runs-on: ubuntu-latest + needs: + - build-repo + + steps: + - name: Install Nix + uses: cachix/install-nix-action@v20 + with: + nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: accept-flake-config = true + + - name: Setup nixbuild.net + uses: nixbuild/nixbuild-action@v16 + with: + nixbuild_token: ${{secrets.NIXBUILD_TOKEN}} + + - uses: actions/checkout@v3 + + - name: Download built repository + uses: actions/download-artifact@v3 + with: + name: built-repo + path: _repo + + - name: Build checks + # The > is the "YAML folded string" marker, which replaces + # newlines with spaces, since the usual bash idiom of \ + # doesn't work for some reason + # + # See https://github.com/nixbuild/feedback/issues/14 for + # why some of these options are here + run: > + nix flake check + -L + --override-input CHaP path:_repo + --eval-store auto + --store ssh-ng://eu.nixbuild.net + --max-jobs 2 + --builders "" + deploy-check: runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: - - build + - build-repo steps: - uses: actions/checkout@v3 @@ -135,7 +176,7 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: - check - - build + - build-repo - deploy-check concurrency: diff --git a/README.md b/README.md index bc3c371d4..7e4364cd8 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ Ideally, include the conditions under which we can deprecate it, e.g. "can depre ## How to test changes to CHaP Sometimes it is useful to test in advance how a new package or a cabal file -revision affects things +revision affects things. The first steps are always the same, you need a built version of your modified CHaP locally: @@ -230,7 +230,7 @@ CHaP locally: For the rest of this section we will assume the built repository is in `/home/user/cardano-haskell-packages/_repo`. -### ... in isolation +### ... by building packages with `cabal` You can test a locally built CHaP with a small test project consisting of just a `cabal.project` file: @@ -259,9 +259,26 @@ Then you can build whatever package version you want with `cabal`: $ cabal build cardano-prelude ``` -You can troubleshoot a failed build plan using the cabal flags `--constraint`, `--allow-newer- and `--allow-older`. Once you have obtained a working build plan, you should revise you cabal file with appropriate constraints. +You can troubleshoot a failed build plan using the cabal flags `--constraint`, `--allow-newer` and `--allow-older`. Once you have obtained a working build plan, you should revise you cabal file with appropriate constraints. -### ... against haskell.nix projects +### ... by building packages with Nix + +You can build packages from CHaP using Nix like this: + +``` +nix build + --override-input CHaP /home/user/cardano-haskell-packages/_repo + .#haskellPackages.x86_64-linux.ghc8107.plutus-core."1.1.0.0" +``` + +This will build all the components of that package version that CHaP cares about, namely libraries and executables (test suites and benchmarks are not built). + +We need to use `--override-input` because the CHaP flake relies on a built repository. +By default it points to a built repository on the main CHaP `repo` branch. +But if you have just produced your own built repository (see above) then you want to +use that instead, and `--override-input` will let you do that. + +### ... by testing against a haskell.nix project If you want to test a locally built CHaP against a project that uses CHaP via haskell.nix, you can build the project while overriding CHaP @@ -310,6 +327,9 @@ The CI for CHaP does the following things: Along with requiring linear history, this ensures that package repository that we build is always an extension of the previous one. - Builds the package repository from the metadata using `foliage`. - Deploys the package repository to the `repo` branch, along with some static web content. +- Builds a small set of packages using the newly built repository, to flush out any build issues. + - We build with all the major GHC versions we expect to be in use. + - At the moment we don't build all the packages in the repository, only the latest versions of a fixed set. ## Creating a repository like CHaP diff --git a/_sources/cardano-crypto-class/2.0.0.1/meta.toml b/_sources/cardano-crypto-class/2.0.0.1/meta.toml index 17965971e..cf9bd004f 100644 --- a/_sources/cardano-crypto-class/2.0.0.1/meta.toml +++ b/_sources/cardano-crypto-class/2.0.0.1/meta.toml @@ -1,3 +1,7 @@ timestamp = 2022-10-17T00:00:00Z github = { repo = "input-output-hk/cardano-base", rev = "4ef911eb7a4d628c938312ff4c92e7c0bb8c83a1" } subdir = 'cardano-crypto-class' + +[[revisions]] +number = 1 +timestamp = 2023-03-15T16:30:48Z diff --git a/_sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal b/_sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal new file mode 100644 index 000000000..1ee4ab3e4 --- /dev/null +++ b/_sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal @@ -0,0 +1,139 @@ +cabal-version: 2.2 +name: cardano-crypto-class +version: 2.0.0.1 +synopsis: + Type classes abstracting over cryptography primitives for Cardano + +description: + Type classes abstracting over cryptography primitives for Cardano + +license: Apache-2.0 +license-files: + LICENSE + NOTICE + +author: IOHK +maintainer: operations@iohk.io +copyright: 2019-2021 IOHK +category: Currency +build-type: Simple +extra-source-files: README.md + +flag development + description: Disable `-Werror` + default: False + manual: True + +flag secp256k1-support + description: Enable support for functions from libsecp256k1. Requires + a recent libsecp256k1 with support for Schnorr signatures. + default: True + manual: True + +common base { build-depends: base >= 4.14 && < 4.17 } + +common project-config + default-language: Haskell2010 + ghc-options: + -Wall -Wcompat -Wincomplete-record-updates + -Wincomplete-uni-patterns -Wpartial-fields -Wredundant-constraints + -Wunused-packages + +library + import: base, project-config + hs-source-dirs: src + exposed-modules: + Cardano.Crypto.DSIGN + Cardano.Crypto.DSIGN.Class + Cardano.Crypto.DSIGN.Ed25519 + Cardano.Crypto.DSIGN.Ed448 + Cardano.Crypto.DSIGN.Mock + Cardano.Crypto.DSIGN.NeverUsed + Cardano.Crypto.Hash + Cardano.Crypto.Hash.Blake2b + Cardano.Crypto.Hash.Class + Cardano.Crypto.Hash.Keccak256 + Cardano.Crypto.Hash.NeverUsed + Cardano.Crypto.Hash.SHA256 + Cardano.Crypto.Hash.SHA3_256 + Cardano.Crypto.Hash.Short + Cardano.Crypto.Init + Cardano.Crypto.KES + Cardano.Crypto.KES.Class + Cardano.Crypto.KES.CompactSingle + Cardano.Crypto.KES.CompactSum + Cardano.Crypto.KES.Mock + Cardano.Crypto.KES.NeverUsed + Cardano.Crypto.KES.Simple + Cardano.Crypto.KES.Single + Cardano.Crypto.KES.Sum + Cardano.Crypto.Libsodium + Cardano.Crypto.Libsodium.C + Cardano.Crypto.Libsodium.Constants + Cardano.Crypto.Libsodium.Hash + Cardano.Crypto.Libsodium.Init + Cardano.Crypto.Libsodium.Memory + Cardano.Crypto.Libsodium.Memory.Internal + Cardano.Crypto.Libsodium.MLockedBytes + Cardano.Crypto.Libsodium.MLockedBytes.Internal + Cardano.Crypto.Libsodium.UnsafeC + Cardano.Crypto.PinnedSizedBytes + Cardano.Crypto.Seed + Cardano.Crypto.Util + Cardano.Crypto.VRF + Cardano.Crypto.VRF.Class + Cardano.Crypto.VRF.Mock + Cardano.Crypto.VRF.NeverUsed + Cardano.Crypto.VRF.Simple + Cardano.Foreign + + other-modules: + Cardano.Crypto.PackedBytes + + build-depends: + , aeson + , base + , base16-bytestring >=1 + , bytestring + , cardano-binary + , cardano-strict-containers + , cryptonite + , deepseq + , heapwords + , memory + , nothunks + , primitive + , serialise + , template-haskell + , th-compat + , text + , transformers + , vector + + if impl(ghc < 9.0.0) + build-depends: + integer-gmp + + pkgconfig-depends: libsodium -any + + if flag(secp256k1-support) + exposed-modules: + Cardano.Crypto.DSIGN.EcdsaSecp256k1 + Cardano.Crypto.DSIGN.SchnorrSecp256k1 + Cardano.Crypto.SECP256K1.Constants + Cardano.Crypto.SECP256K1.C + pkgconfig-depends: libsecp256k1 -any + cpp-options: -DSECP256K1_ENABLED + +test-suite test-memory-example + import: base, project-config + type: exitcode-stdio-1.0 + hs-source-dirs: memory-example + main-is: Main.hs + build-depends: + , base + , bytestring + , cardano-crypto-class + + if (os(linux) || os(osx)) + build-depends: unix diff --git a/builder/default.nix b/builder/default.nix deleted file mode 100644 index b55cc49ae..000000000 --- a/builder/default.nix +++ /dev/null @@ -1,56 +0,0 @@ -{ pkgs, CHaP }: -compiler-nix-name: -let - inherit (pkgs) lib; - - # last index state known to haskell-nix - index-state = lib.last (builtins.attrNames (import pkgs.haskell-nix.indexStateHashesPath)); - - chap-package-list = - builtins.filter (lib.strings.hasPrefix "plutus-core") ( - builtins.map (p: "${p.pkg-name}-${p.pkg-version}") - (builtins.fromJSON (builtins.readFile "${CHaP}/foliage/packages.json"))); - - build-chap-package = package-id: - let - package-name = (builtins.parseDrvName package-id).name; - - project = pkgs.haskell-nix.cabalProject' { - inherit compiler-nix-name; - inherit index-state; - - name = package-id; - src = ./empty; - - inputMap = { - "https://input-output-hk.github.io/cardano-haskell-packages" = CHaP; - }; - - cabalProject = '' - repository cardano-haskell-packages - url: https://input-output-hk.github.io/cardano-haskell-packages - secure: True - extra-packages: ${package-id} - ''; - - modules = [{ - packages = { - cardano-crypto-praos.components.library.pkgconfig = lib.mkForce [ [ pkgs.libsodium-vrf pkgs.secp256k1 ] ]; - cardano-crypto-class.components.library.pkgconfig = lib.mkForce [ [ pkgs.libsodium-vrf pkgs.secp256k1 ] ]; - }; - }]; - - }; - - in - pkgs.releaseTools.aggregate { - name = package-id; - constituents = lib.collect lib.isDerivation { - inherit (project.hsPkgs.${package-name}) components checks; - }; - }; - -in -lib.attrsets.mapAttrs' (name: lib.attrsets.nameValuePair (builtins.replaceStrings [ "." ] [ "-" ] name)) ( - lib.attrsets.genAttrs chap-package-list build-chap-package -) diff --git a/flake.lock b/flake.lock index 09d291df8..e08053ffb 100644 --- a/flake.lock +++ b/flake.lock @@ -316,11 +316,11 @@ "hackage-nix": { "flake": false, "locked": { - "lastModified": 1678839912, - "narHash": "sha256-U13Gt95c4lAy+J1tAEInK4batuv4Az3q2Imar8V4qqA=", + "lastModified": 1679012701, + "narHash": "sha256-qWldqfEJEkqhQGEbx5yeZ+huc2muYj8WrMKx+uFqgMo=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "8d86d399acebee778bc0211008d0f1cea77b4307", + "rev": "a618dbd152ae5a8de7978ea5655096f7846cb880", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index a3f9b3254..f84e19e1a 100644 --- a/flake.nix +++ b/flake.nix @@ -34,18 +34,36 @@ }; outputs = { self, nixpkgs, flake-utils, foliage, haskell-nix, CHaP, iohk-nix, ... }: - flake-utils.lib.eachDefaultSystem + # The foliage flake only works on this system, so we are stuck with it too + flake-utils.lib.eachSystem [ "x86_64-linux" ] (system: - let pkgs = import nixpkgs { - inherit system; - inherit (haskell-nix) config; - overlays = [ - haskell-nix.overlay - iohk-nix.overlays.crypto - ]; - }; + let + pkgs = import nixpkgs { + inherit system; + inherit (haskell-nix) config; + overlays = [ + haskell-nix.overlay + iohk-nix.overlays.crypto + ]; + }; + inherit (pkgs) lib; + + compilers = [ "ghc8107" "ghc926" ]; + + builder = import ./nix/builder.nix { inherit pkgs CHaP; }; + chap-meta = import ./nix/chap-meta.nix { inherit pkgs CHaP; }; + + smokeTestPackages = [ + #"cardano-node" + #"cardano-cli" + #"cardano-api" + "plutus-core" + ]; + # using intersectAttrs like this is a cheap way to throw away everything with keys not in + # smokeTestPackages + smokeTestPackageVersions = builtins.intersectAttrs (lib.genAttrs smokeTestPackages (pkg: {})) chap-meta.chap-package-latest-versions; in - { + rec { devShells.default = with pkgs; mkShellNoCC { name = "cardano-haskell-packages"; buildInputs = [ @@ -58,13 +76,25 @@ ]; }; - hydraJobs = + # { ghc8107 = { foo = { X.Y.Z = ; ... }; ...}; ... } + haskellPackages = let - inherit (pkgs) lib; - builder = import ./builder { inherit pkgs CHaP; }; - compilers = [ "ghc8107" "ghc925" ]; + derivations = compiler: (lib.mapAttrs + (name: versions: (lib.genAttrs versions (version: builder compiler name version))) + chap-meta.chap-package-versions); in - lib.attrsets.genAttrs compilers builder; + lib.genAttrs compilers derivations; + + # The standard checks: build all the smoke test packages + checks = + # We use recurseIntoAttrs so flattenTree will flatten it back out again. + let + derivations = compiler: lib.recurseIntoAttrs (lib.mapAttrs (name: version: builder compiler name version) smokeTestPackageVersions); + # A nested tree of derivations containing all the smoke test packages for all the compiler versions + perCompilerDerivations = lib.recurseIntoAttrs (lib.genAttrs compilers derivations); + in builtins.trace (builtins.toJSON smokeTestPackageVersions) (flake-utils.lib.flattenTree perCompilerDerivations); + + hydraJobs = checks; }); nixConfig = { @@ -73,10 +103,18 @@ "https://foliage.cachix.org" "https://cache.zw3rk.com" ]; + extra-trusted-substituters = [ + # If you have a nixbuild.net SSH key set up, you can pull builds from there + # by using '--extra-substituters ssh://eu.nixbuild.net' manually, otherwise this + # does nothing + "ssh://eu.nixbuild.net" + ]; extra-trusted-public-keys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" "foliage.cachix.org-1:kAFyYLnk8JcRURWReWZCatM9v3Rk24F5wNMpEj14Q/g=" "loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=" + "nixbuild.net/smart.contracts@iohk.io-1:s2PhQXWwsZo1y5IxFcx2D/i2yfvgtEnRBOZavlA8Bog=" ]; + allow-import-from-derivation = true; }; } diff --git a/nix/builder.nix b/nix/builder.nix new file mode 100644 index 000000000..f4da1299f --- /dev/null +++ b/nix/builder.nix @@ -0,0 +1,58 @@ +{ pkgs, CHaP }: +compiler-nix-name: +let + inherit (pkgs) lib; + inherit (pkgs.haskell-nix) haskellLib; + + build-chap-package = package-name: package-version: + let + package-id = "${package-name}-${package-version}"; + + # Global config needed to build CHaP packages should go here. Obviously + # this should be kept to an absolute minimum, since that means config + # that every downstream project needs also. + # + # No need to set index-state: + # - haskell.nix will automatically use the latest known one for hackage + # - we want the very latest state for CHaP so it includes anything from + # e.g. a PR being tested + project = pkgs.haskell-nix.cabalProject' { + inherit compiler-nix-name; + + name = package-id; + src = ./empty; + + inputMap = { + "https://input-output-hk.github.io/cardano-haskell-packages" = CHaP; + }; + + # Note that we do not set tests or benchmarks to True, so we won't + # build them by default. This is the same as what happens on Hackage, + # for example, and they can't be depended on by downstream packages + # anyway. + cabalProject = '' + repository cardano-haskell-packages + url: https://input-output-hk.github.io/cardano-haskell-packages + secure: True + + extra-packages: ${package-id} + ''; + + modules = [{ + packages = { + cardano-crypto-praos.components.library.pkgconfig = lib.mkForce [ [ pkgs.libsodium-vrf pkgs.secp256k1 ] ]; + cardano-crypto-class.components.library.pkgconfig = lib.mkForce [ [ pkgs.libsodium-vrf pkgs.secp256k1 ] ]; + }; + }]; + + }; + pkg = project.hsPkgs.${package-name}; + components = haskellLib.getAllComponents pkg; + in pkgs.releaseTools.aggregate { + name = package-id; + # Note that this does *not* include the derivations from 'checks' which + # actually run tests: CHaP will not check that your tests pass (neither + # does Hackage). + constituents = components; + }; +in build-chap-package diff --git a/nix/chap-meta.nix b/nix/chap-meta.nix new file mode 100644 index 000000000..42f8040b0 --- /dev/null +++ b/nix/chap-meta.nix @@ -0,0 +1,29 @@ +{ pkgs, CHaP }: +let + inherit (pkgs) lib; + inherit (pkgs.haskell-nix) haskellLib; + + + # [ { name = "foo"; version = "X.Y.Z"; }, { name = "foo"; version = "P.Q.R"; } ] + chap-package-list = + builtins.map (p: { name = p.pkg-name; version = p.pkg-version; }) + (builtins.fromJSON (builtins.readFile "${CHaP}/foliage/packages.json")); + + # { "foo" : [ "X.Y.Z" "P.Q.R" ], ... } + chap-package-versions = + let + # { "foo" = [{ name = "foo"; version = "X.Y.Z"; }, { name = "foo"; version = "P.Q.R"; }]; ... } + grouped = lib.groupBy (m: m.name) chap-package-list; + # { "foo" : [ "X.Y.Z" "P.Q.R" ], ... } + simplified = lib.mapAttrs (name: metas: builtins.map (m: m.version) metas) grouped; + in simplified; + + # { "foo" : "X.Y.Z", ... } + chap-package-latest-versions = + let latest = versions: lib.last (lib.naturalSort versions); + in lib.mapAttrs (name: versions: latest versions) chap-package-versions; + +in +{ + inherit chap-package-versions chap-package-latest-versions; +} diff --git a/builder/empty/almost/.gitkeep b/nix/empty/almost/.gitkeep similarity index 100% rename from builder/empty/almost/.gitkeep rename to nix/empty/almost/.gitkeep