From af9c8b585d916b99769f627252f0910f33edff3d Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Tue, 14 Mar 2023 14:31:56 +0000 Subject: [PATCH 01/18] CI attempt 2 --- .github/workflows/CI.yml | 55 +++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8ed33893a..4dc6259b5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 # the check script below needs the whole history - name: Setup Nix - uses: cachix/install-nix-action@v17 + uses: cachix/install-nix-action@v20 with: nix_path: nixpkgs=channel:nixos-unstable extra_nix_config: | @@ -29,12 +29,12 @@ jobs: - name: Run checks run: nix develop -c ./scripts/check.sh - build: + build-repo: runs-on: ubuntu-latest steps: - name: Setup Nix - uses: cachix/install-nix-action@v17 + uses: cachix/install-nix-action@v20 with: nix_path: nixpkgs=channel:nixos-unstable extra_nix_config: | @@ -64,8 +64,10 @@ jobs: fi - name: Build repository (main) + # We don't need the metadata here, since we're just + # using this to compare the generated index run: | - nix develop -c foliage build -j 0 + nix develop -c foliage build -j 0 mv _repo _repo-main - name: Checkout tip commit @@ -84,7 +86,7 @@ jobs: # Do this before the check, useful to have the artifact in case the # check fails! - - name: Upload artifact + - name: Upload built repository uses: actions/upload-artifact@v3 with: name: built-repo @@ -100,11 +102,50 @@ jobs: echo "then you may need to update the timestamps in your new packages to be newer than those in main." ./scripts/check-archive-extension.sh _repo-main/01-index.tar _repo/01-index.tar + build-packages: + runs-on: ubuntu-latest + needs: + - build-repo + strategy: + matrix: + ghc: + - ghc8107 + - ghc925 + + steps: + - name: Install Nix + uses: cachix/install-nix-action@v20 + with: + nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + substituters = https://cache.iog.io https://cache.nixos.org https://foliage.cachix.org + trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= foliage.cachix.org-1:kAFyYLnk8JcRURWReWZCatM9v3Rk24F5wNMpEj14Q/g= + + - name: Setup nixbuild.net + uses: nixbuild/nixbuild-action@biscuit + with: + nixbuild_token: ${{secrets.NIXBUILD_TOKEN}} + + - uses: actions/checkout@v3 + with: + # The flake for building packages is on the builder branch + ref: builder + + - name: Download built repository + uses: actions/download-artifact@v3 + with: + name: built-repo + path: _repo + + - name: Build a smoke-test package + run: | + nix build --accept-flake-config -L --override-input CHaP path:_repo --eval-store auto --store ssh-ng://eu.nixbuild.net .#hydraJobs.${{matrix.ghc}}.plutus-core-1-1-1-0.x86_64-linux + deploy-check: runs-on: ubuntu-latest if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: - - build + - build-repo steps: - uses: actions/checkout@v3 @@ -135,7 +176,7 @@ jobs: if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: - check - - build + - build-repo - deploy-check concurrency: From 081fc5e4d1506b1cb856ec934f7f0732c22c6fd3 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 15 Mar 2023 16:31:36 +0000 Subject: [PATCH 02/18] Revise cardano-crypto-class --- .../cardano-crypto-class/2.0.0.1/meta.toml | 4 + .../2.0.0.1/revisions/1.cabal | 139 ++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 _sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal diff --git a/_sources/cardano-crypto-class/2.0.0.1/meta.toml b/_sources/cardano-crypto-class/2.0.0.1/meta.toml index 17965971e..cf9bd004f 100644 --- a/_sources/cardano-crypto-class/2.0.0.1/meta.toml +++ b/_sources/cardano-crypto-class/2.0.0.1/meta.toml @@ -1,3 +1,7 @@ timestamp = 2022-10-17T00:00:00Z github = { repo = "input-output-hk/cardano-base", rev = "4ef911eb7a4d628c938312ff4c92e7c0bb8c83a1" } subdir = 'cardano-crypto-class' + +[[revisions]] +number = 1 +timestamp = 2023-03-15T16:30:48Z diff --git a/_sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal b/_sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal new file mode 100644 index 000000000..1ee4ab3e4 --- /dev/null +++ b/_sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal @@ -0,0 +1,139 @@ +cabal-version: 2.2 +name: cardano-crypto-class +version: 2.0.0.1 +synopsis: + Type classes abstracting over cryptography primitives for Cardano + +description: + Type classes abstracting over cryptography primitives for Cardano + +license: Apache-2.0 +license-files: + LICENSE + NOTICE + +author: IOHK +maintainer: operations@iohk.io +copyright: 2019-2021 IOHK +category: Currency +build-type: Simple +extra-source-files: README.md + +flag development + description: Disable `-Werror` + default: False + manual: True + +flag secp256k1-support + description: Enable support for functions from libsecp256k1. Requires + a recent libsecp256k1 with support for Schnorr signatures. + default: True + manual: True + +common base { build-depends: base >= 4.14 && < 4.17 } + +common project-config + default-language: Haskell2010 + ghc-options: + -Wall -Wcompat -Wincomplete-record-updates + -Wincomplete-uni-patterns -Wpartial-fields -Wredundant-constraints + -Wunused-packages + +library + import: base, project-config + hs-source-dirs: src + exposed-modules: + Cardano.Crypto.DSIGN + Cardano.Crypto.DSIGN.Class + Cardano.Crypto.DSIGN.Ed25519 + Cardano.Crypto.DSIGN.Ed448 + Cardano.Crypto.DSIGN.Mock + Cardano.Crypto.DSIGN.NeverUsed + Cardano.Crypto.Hash + Cardano.Crypto.Hash.Blake2b + Cardano.Crypto.Hash.Class + Cardano.Crypto.Hash.Keccak256 + Cardano.Crypto.Hash.NeverUsed + Cardano.Crypto.Hash.SHA256 + Cardano.Crypto.Hash.SHA3_256 + Cardano.Crypto.Hash.Short + Cardano.Crypto.Init + Cardano.Crypto.KES + Cardano.Crypto.KES.Class + Cardano.Crypto.KES.CompactSingle + Cardano.Crypto.KES.CompactSum + Cardano.Crypto.KES.Mock + Cardano.Crypto.KES.NeverUsed + Cardano.Crypto.KES.Simple + Cardano.Crypto.KES.Single + Cardano.Crypto.KES.Sum + Cardano.Crypto.Libsodium + Cardano.Crypto.Libsodium.C + Cardano.Crypto.Libsodium.Constants + Cardano.Crypto.Libsodium.Hash + Cardano.Crypto.Libsodium.Init + Cardano.Crypto.Libsodium.Memory + Cardano.Crypto.Libsodium.Memory.Internal + Cardano.Crypto.Libsodium.MLockedBytes + Cardano.Crypto.Libsodium.MLockedBytes.Internal + Cardano.Crypto.Libsodium.UnsafeC + Cardano.Crypto.PinnedSizedBytes + Cardano.Crypto.Seed + Cardano.Crypto.Util + Cardano.Crypto.VRF + Cardano.Crypto.VRF.Class + Cardano.Crypto.VRF.Mock + Cardano.Crypto.VRF.NeverUsed + Cardano.Crypto.VRF.Simple + Cardano.Foreign + + other-modules: + Cardano.Crypto.PackedBytes + + build-depends: + , aeson + , base + , base16-bytestring >=1 + , bytestring + , cardano-binary + , cardano-strict-containers + , cryptonite + , deepseq + , heapwords + , memory + , nothunks + , primitive + , serialise + , template-haskell + , th-compat + , text + , transformers + , vector + + if impl(ghc < 9.0.0) + build-depends: + integer-gmp + + pkgconfig-depends: libsodium -any + + if flag(secp256k1-support) + exposed-modules: + Cardano.Crypto.DSIGN.EcdsaSecp256k1 + Cardano.Crypto.DSIGN.SchnorrSecp256k1 + Cardano.Crypto.SECP256K1.Constants + Cardano.Crypto.SECP256K1.C + pkgconfig-depends: libsecp256k1 -any + cpp-options: -DSECP256K1_ENABLED + +test-suite test-memory-example + import: base, project-config + type: exitcode-stdio-1.0 + hs-source-dirs: memory-example + main-is: Main.hs + build-depends: + , base + , bytestring + , cardano-crypto-class + + if (os(linux) || os(osx)) + build-depends: unix From 58ab9143f2cc5d30b8aae98c898bf36893a18f1e Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 15 Mar 2023 17:17:29 +0000 Subject: [PATCH 03/18] Don't check out builder any more --- .github/workflows/CI.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4dc6259b5..7ecabb08a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -127,9 +127,6 @@ jobs: nixbuild_token: ${{secrets.NIXBUILD_TOKEN}} - uses: actions/checkout@v3 - with: - # The flake for building packages is on the builder branch - ref: builder - name: Download built repository uses: actions/download-artifact@v3 From f7075ce42d127a22982e478cbeba00435d36f0f8 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 15 Mar 2023 17:31:56 +0000 Subject: [PATCH 04/18] Small fixes for the builder --- builder/default.nix | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/builder/default.nix b/builder/default.nix index b55cc49ae..1e583e37a 100644 --- a/builder/default.nix +++ b/builder/default.nix @@ -3,9 +3,6 @@ compiler-nix-name: let inherit (pkgs) lib; - # last index state known to haskell-nix - index-state = lib.last (builtins.attrNames (import pkgs.haskell-nix.indexStateHashesPath)); - chap-package-list = builtins.filter (lib.strings.hasPrefix "plutus-core") ( builtins.map (p: "${p.pkg-name}-${p.pkg-version}") @@ -15,9 +12,12 @@ let let package-name = (builtins.parseDrvName package-id).name; + # No need to set index-state: + # - haskell.nix will automatically use the latest known one for hackage + # - we want the very latest state for CHaP so it includes anything from + # e.g. a PR being tested project = pkgs.haskell-nix.cabalProject' { inherit compiler-nix-name; - inherit index-state; name = package-id; src = ./empty; @@ -30,6 +30,11 @@ let repository cardano-haskell-packages url: https://input-output-hk.github.io/cardano-haskell-packages secure: True + + -- Work around https://github.com/input-output-hk/nothunks/issues/17 + package nothunks + flags: +vector + extra-packages: ${package-id} ''; From c0664a8bc38bc092b0a9b4f31372e490fcc8a1b4 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 15 Mar 2023 17:48:46 +0000 Subject: [PATCH 05/18] later compiler --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index a3f9b3254..0f1be312c 100644 --- a/flake.nix +++ b/flake.nix @@ -62,7 +62,7 @@ let inherit (pkgs) lib; builder = import ./builder { inherit pkgs CHaP; }; - compilers = [ "ghc8107" "ghc925" ]; + compilers = [ "ghc8107" "ghc926" ]; in lib.attrsets.genAttrs compilers builder; }); From bfbfda280166090b92a3f8d1558e3ba395707b40 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 15 Mar 2023 17:55:25 +0000 Subject: [PATCH 06/18] Try this --- .github/workflows/CI.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7ecabb08a..c02059a22 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -118,8 +118,7 @@ jobs: with: nix_path: nixpkgs=channel:nixos-unstable extra_nix_config: | - substituters = https://cache.iog.io https://cache.nixos.org https://foliage.cachix.org - trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= foliage.cachix.org-1:kAFyYLnk8JcRURWReWZCatM9v3Rk24F5wNMpEj14Q/g= + accept-flake-config = true - name: Setup nixbuild.net uses: nixbuild/nixbuild-action@biscuit From 92b29fa8effaee4cddf311dc08996efd41bbd73a Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Wed, 15 Mar 2023 18:04:28 +0000 Subject: [PATCH 07/18] More new compiler --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c02059a22..1bb61f676 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -110,7 +110,7 @@ jobs: matrix: ghc: - ghc8107 - - ghc925 + - ghc926 steps: - name: Install Nix From b3637c6a0930300e1d5cc6864c699c717217c510 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Thu, 16 Mar 2023 09:39:00 +0000 Subject: [PATCH 08/18] Try multiline again --- .github/workflows/CI.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1bb61f676..c73db13f5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -134,8 +134,17 @@ jobs: path: _repo - name: Build a smoke-test package - run: | - nix build --accept-flake-config -L --override-input CHaP path:_repo --eval-store auto --store ssh-ng://eu.nixbuild.net .#hydraJobs.${{matrix.ghc}}.plutus-core-1-1-1-0.x86_64-linux + # This is the "YAML folded string" marker, which replaces + # newlines with spaces, since the usual bash idiom of \ + # doesn't work for some reason + run: > + nix build + --accept-flake-config + -L + --override-input CHaP path:_repo + --eval-store auto + --store ssh-ng://eu.nixbuild.net + .#hydraJobs.${{matrix.ghc}}.plutus-core-1-1-1-0.x86_64-linux deploy-check: runs-on: ubuntu-latest From 9d046c004d53c9ff20ee7b2ebf5eddb301b6e4c0 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Thu, 16 Mar 2023 09:39:33 +0000 Subject: [PATCH 09/18] Add some extra config for IFD --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index c73db13f5..913917f70 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -144,6 +144,8 @@ jobs: --override-input CHaP path:_repo --eval-store auto --store ssh-ng://eu.nixbuild.net + --max-jobs 2 + --builders "" .#hydraJobs.${{matrix.ghc}}.plutus-core-1-1-1-0.x86_64-linux deploy-check: From 9cf00015d3e34148426e0103118596e313ca63d5 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Thu, 16 Mar 2023 11:32:52 +0000 Subject: [PATCH 10/18] Tweaks --- .github/workflows/CI.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 913917f70..3a76b1eaf 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -134,12 +134,14 @@ jobs: path: _repo - name: Build a smoke-test package - # This is the "YAML folded string" marker, which replaces + # The > is the "YAML folded string" marker, which replaces # newlines with spaces, since the usual bash idiom of \ # doesn't work for some reason + # + # See https://github.com/nixbuild/feedback/issues/14 for + # why some of these options are here run: > nix build - --accept-flake-config -L --override-input CHaP path:_repo --eval-store auto From a60768631b4a505887769fed223efb42774fcd4a Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Thu, 16 Mar 2023 11:34:32 +0000 Subject: [PATCH 11/18] Add nixbuild.net as a substituter --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 0f1be312c..9a5f07fd0 100644 --- a/flake.nix +++ b/flake.nix @@ -73,10 +73,14 @@ "https://foliage.cachix.org" "https://cache.zw3rk.com" ]; + extra-trusted-substituters = [ + "ssh://eu.nixbuild.net" + ]; extra-trusted-public-keys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" "foliage.cachix.org-1:kAFyYLnk8JcRURWReWZCatM9v3Rk24F5wNMpEj14Q/g=" "loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=" + "nixbuild.net/smart.contracts@iohk.io-1:s2PhQXWwsZo1y5IxFcx2D/i2yfvgtEnRBOZavlA8Bog=" ]; }; } From 5a43ce36168ae118cdf76f8173adfc92d4c7b648 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Thu, 16 Mar 2023 15:37:44 +0000 Subject: [PATCH 12/18] Major flake restructuring --- .github/workflows/CI.yml | 10 ++------ builder/default.nix | 38 +++++++++++++++------------- flake.nix | 54 +++++++++++++++++++++++++++++----------- 3 files changed, 63 insertions(+), 39 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3a76b1eaf..243d2f110 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -106,11 +106,6 @@ jobs: runs-on: ubuntu-latest needs: - build-repo - strategy: - matrix: - ghc: - - ghc8107 - - ghc926 steps: - name: Install Nix @@ -133,7 +128,7 @@ jobs: name: built-repo path: _repo - - name: Build a smoke-test package + - name: Build checks # The > is the "YAML folded string" marker, which replaces # newlines with spaces, since the usual bash idiom of \ # doesn't work for some reason @@ -141,14 +136,13 @@ jobs: # See https://github.com/nixbuild/feedback/issues/14 for # why some of these options are here run: > - nix build + nix flake check -L --override-input CHaP path:_repo --eval-store auto --store ssh-ng://eu.nixbuild.net --max-jobs 2 --builders "" - .#hydraJobs.${{matrix.ghc}}.plutus-core-1-1-1-0.x86_64-linux deploy-check: runs-on: ubuntu-latest diff --git a/builder/default.nix b/builder/default.nix index 1e583e37a..6a071fd2c 100644 --- a/builder/default.nix +++ b/builder/default.nix @@ -2,15 +2,25 @@ compiler-nix-name: let inherit (pkgs) lib; + inherit (pkgs.haskell-nix) haskellLib; + # [ { name = "foo"; version = "X.Y.Z"; }, { name = "foo"; version = "P.Q.R"; } ] chap-package-list = - builtins.filter (lib.strings.hasPrefix "plutus-core") ( - builtins.map (p: "${p.pkg-name}-${p.pkg-version}") - (builtins.fromJSON (builtins.readFile "${CHaP}/foliage/packages.json"))); + builtins.map (p: { name = p.pkg-name; version = p.pkg-version; }) + (builtins.fromJSON (builtins.readFile "${CHaP}/foliage/packages.json")); - build-chap-package = package-id: + # { "foo" : [ "X.Y.Z" "P.Q.R" ], ... } + chap-package-attrs = + let + # { "foo" = [{ name = "foo"; version = "X.Y.Z"; }, { name = "foo"; version = "P.Q.R"; }]; ... } + grouped = lib.groupBy (m: m.name) chap-package-list; + # { "foo" : [ "X.Y.Z" "P.Q.R" ], ... } + simplified = lib.mapAttrs (name: metas: builtins.map (m: m.version) metas) grouped; + in simplified; + + chap-package-components = package-name: package-version: let - package-name = (builtins.parseDrvName package-id).name; + package-id = "${package-name}-${package-version}"; # No need to set index-state: # - haskell.nix will automatically use the latest known one for hackage @@ -46,16 +56,10 @@ let }]; }; - - in - pkgs.releaseTools.aggregate { - name = package-id; - constituents = lib.collect lib.isDerivation { - inherit (project.hsPkgs.${package-name}) components checks; - }; - }; - + pkg = project.hsPkgs.${package-name}; + components = haskellLib.getAllComponents pkg; + # Not an ideal name here, would be nice to use something simpler + in lib.recurseIntoAttrs (builtins.listToAttrs (builtins.map (c: lib.nameValuePair c.name c) components)); in -lib.attrsets.mapAttrs' (name: lib.attrsets.nameValuePair (builtins.replaceStrings [ "." ] [ "-" ] name)) ( - lib.attrsets.genAttrs chap-package-list build-chap-package -) +# { foo = { X.Y.Z = ; P.Q.R = ; }; ... } +lib.recurseIntoAttrs (lib.mapAttrs (name: versions: lib.recurseIntoAttrs (lib.genAttrs versions (version: chap-package-components name version))) chap-package-attrs) diff --git a/flake.nix b/flake.nix index 9a5f07fd0..784f7eb5e 100644 --- a/flake.nix +++ b/flake.nix @@ -34,18 +34,29 @@ }; outputs = { self, nixpkgs, flake-utils, foliage, haskell-nix, CHaP, iohk-nix, ... }: - flake-utils.lib.eachDefaultSystem + let + compilers = [ "ghc8107" "ghc926" ]; + smokeTestPackages = [ + "cardano-node" + "cardano-cli" + "cardano-api" + "plutus-core" + ]; + # The foliage flake only works on this system, so we are stuck with it too + in flake-utils.lib.eachSystem [ "x86_64-linux" ] (system: - let pkgs = import nixpkgs { - inherit system; - inherit (haskell-nix) config; - overlays = [ - haskell-nix.overlay - iohk-nix.overlays.crypto - ]; - }; + let + pkgs = import nixpkgs { + inherit system; + inherit (haskell-nix) config; + overlays = [ + haskell-nix.overlay + iohk-nix.overlays.crypto + ]; + }; + inherit (pkgs) lib; in - { + rec { devShells.default = with pkgs; mkShellNoCC { name = "cardano-haskell-packages"; buildInputs = [ @@ -58,13 +69,27 @@ ]; }; - hydraJobs = + # { ghc8107 = { foo = { X.Y.Z = ; ... }; ...}; ... } + haskellPackages = let - inherit (pkgs) lib; builder = import ./builder { inherit pkgs CHaP; }; - compilers = [ "ghc8107" "ghc926" ]; in - lib.attrsets.genAttrs compilers builder; + lib.recurseIntoAttrs (lib.genAttrs compilers builder); + + # A nested tree of derivations containing all the smoke test packages for all the compiler versions + smokeTestDerivations = lib.genAttrs compilers (compiler: + let + # The latest version in the set (attrValues sorts by key). Remove the 'recurseForDerivations' here otherwise + # we get that! + latest = versionToValue: lib.last (lib.attrValues (builtins.removeAttrs versionToValue ["recurseForDerivations"])); + compilerPkgs = builtins.getAttr compiler haskellPackages; + smokeTestPkgs = lib.recurseIntoAttrs (lib.genAttrs smokeTestPackages (pkgname: latest (builtins.getAttr pkgname compilerPkgs))); + in smokeTestPkgs); + + # The standard checks: build all the smoke test packages + checks = flake-utils.lib.flattenTree smokeTestDerivations; + + hydraJobs = checks; }); nixConfig = { @@ -82,5 +107,6 @@ "loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=" "nixbuild.net/smart.contracts@iohk.io-1:s2PhQXWwsZo1y5IxFcx2D/i2yfvgtEnRBOZavlA8Bog=" ]; + allow-import-from-derivation = true; }; } From 42fa19edddefee7061f2ee829fae09c8efe7398c Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Thu, 16 Mar 2023 15:57:04 +0000 Subject: [PATCH 13/18] Just plutus-core for now --- flake.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 784f7eb5e..8ca469c3a 100644 --- a/flake.nix +++ b/flake.nix @@ -37,9 +37,9 @@ let compilers = [ "ghc8107" "ghc926" ]; smokeTestPackages = [ - "cardano-node" - "cardano-cli" - "cardano-api" + #"cardano-node" + #"cardano-cli" + #"cardano-api" "plutus-core" ]; # The foliage flake only works on this system, so we are stuck with it too From 7ec3e1b8499ac6ba47ee993ed89a9582e36c2b6f Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Thu, 16 Mar 2023 16:36:52 +0000 Subject: [PATCH 14/18] Doc --- README.md | 26 +++++++++++++++++++++++--- builder/default.nix | 3 +-- flake.nix | 2 ++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bc3c371d4..6b27dccd0 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ Ideally, include the conditions under which we can deprecate it, e.g. "can depre ## How to test changes to CHaP Sometimes it is useful to test in advance how a new package or a cabal file -revision affects things +revision affects things. The first steps are always the same, you need a built version of your modified CHaP locally: @@ -230,7 +230,7 @@ CHaP locally: For the rest of this section we will assume the built repository is in `/home/user/cardano-haskell-packages/_repo`. -### ... in isolation +### ... by building packages with Cabal You can test a locally built CHaP with a small test project consisting of just a `cabal.project` file: @@ -261,7 +261,24 @@ $ cabal build cardano-prelude You can troubleshoot a failed build plan using the cabal flags `--constraint`, `--allow-newer- and `--allow-older`. Once you have obtained a working build plan, you should revise you cabal file with appropriate constraints. -### ... against haskell.nix projects +### ... by building packages with Nix + +You can build packages from CHaP using Nix like this: + +``` +nix build + --override-input /home/user/cardano-haskell-packages/_repo + .#haskellPackages.x86_64-linux.ghc8107.plutus-core."1.1.0.0".plutus-core-exe-plc +``` + +The final attribute name is `--`. + +We need to use `--override-input` because the CHaP flake relies on a built repository. +By default it points to a built repository on the main CHaP `repo` branch. +But if you have just produced your own built repository (see above) then you want to +use that instead, and `-override-input` will let you do that. + +### ... by testing against a haskell.nix project If you want to test a locally built CHaP against a project that uses CHaP via haskell.nix, you can build the project while overriding CHaP @@ -310,6 +327,9 @@ The CI for CHaP does the following things: Along with requiring linear history, this ensures that package repository that we build is always an extension of the previous one. - Builds the package repository from the metadata using `foliage`. - Deploys the package repository to the `repo` branch, along with some static web content. +- Builds a small set of packages using the newly built repository, to flush out any build issues. + - We build with all the major GHC versions we expect to be in use. + - At the moment we don't build all the packages in the repository, only a fixed set. ## Creating a repository like CHaP diff --git a/builder/default.nix b/builder/default.nix index 6a071fd2c..2e74fa24f 100644 --- a/builder/default.nix +++ b/builder/default.nix @@ -58,8 +58,7 @@ let }; pkg = project.hsPkgs.${package-name}; components = haskellLib.getAllComponents pkg; - # Not an ideal name here, would be nice to use something simpler - in lib.recurseIntoAttrs (builtins.listToAttrs (builtins.map (c: lib.nameValuePair c.name c) components)); + in lib.recurseIntoAttrs (builtins.listToAttrs (builtins.map (c: lib.nameValuePair c.pname c) components)); in # { foo = { X.Y.Z = ; P.Q.R = ; }; ... } lib.recurseIntoAttrs (lib.mapAttrs (name: versions: lib.recurseIntoAttrs (lib.genAttrs versions (version: chap-package-components name version))) chap-package-attrs) diff --git a/flake.nix b/flake.nix index 8ca469c3a..ea04c419e 100644 --- a/flake.nix +++ b/flake.nix @@ -99,6 +99,8 @@ "https://cache.zw3rk.com" ]; extra-trusted-substituters = [ + # If you have a nixbuild.net SSH key set up, you can pull builds from there + # by using '--extra-substituters ssh://eu.nixbuild.net' manually "ssh://eu.nixbuild.net" ]; extra-trusted-public-keys = [ From 8cd90b38e641ad676072a2a727249caa0b1402b3 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Fri, 17 Mar 2023 10:05:20 +0000 Subject: [PATCH 15/18] Comments --- .github/workflows/CI.yml | 9 +++------ README.md | 8 ++++---- builder/default.nix | 20 ++++++++++++++++---- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 243d2f110..8dd435af5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,8 +23,7 @@ jobs: uses: cachix/install-nix-action@v20 with: nix_path: nixpkgs=channel:nixos-unstable - extra_nix_config: | - accept-flake-config = true + extra_nix_config: accept-flake-config = true - name: Run checks run: nix develop -c ./scripts/check.sh @@ -37,8 +36,7 @@ jobs: uses: cachix/install-nix-action@v20 with: nix_path: nixpkgs=channel:nixos-unstable - extra_nix_config: | - accept-flake-config = true + extra_nix_config: accept-flake-config = true - name: Checkout main uses: actions/checkout@v3 @@ -112,8 +110,7 @@ jobs: uses: cachix/install-nix-action@v20 with: nix_path: nixpkgs=channel:nixos-unstable - extra_nix_config: | - accept-flake-config = true + extra_nix_config: accept-flake-config = true - name: Setup nixbuild.net uses: nixbuild/nixbuild-action@biscuit diff --git a/README.md b/README.md index 6b27dccd0..ba645d668 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ CHaP locally: For the rest of this section we will assume the built repository is in `/home/user/cardano-haskell-packages/_repo`. -### ... by building packages with Cabal +### ... by building packages with `cabal` You can test a locally built CHaP with a small test project consisting of just a `cabal.project` file: @@ -259,7 +259,7 @@ Then you can build whatever package version you want with `cabal`: $ cabal build cardano-prelude ``` -You can troubleshoot a failed build plan using the cabal flags `--constraint`, `--allow-newer- and `--allow-older`. Once you have obtained a working build plan, you should revise you cabal file with appropriate constraints. +You can troubleshoot a failed build plan using the cabal flags `--constraint`, `--allow-newer` and `--allow-older`. Once you have obtained a working build plan, you should revise you cabal file with appropriate constraints. ### ... by building packages with Nix @@ -267,7 +267,7 @@ You can build packages from CHaP using Nix like this: ``` nix build - --override-input /home/user/cardano-haskell-packages/_repo + --override-input CHaP /home/user/cardano-haskell-packages/_repo .#haskellPackages.x86_64-linux.ghc8107.plutus-core."1.1.0.0".plutus-core-exe-plc ``` @@ -276,7 +276,7 @@ The final attribute name is `--`. We need to use `--override-input` because the CHaP flake relies on a built repository. By default it points to a built repository on the main CHaP `repo` branch. But if you have just produced your own built repository (see above) then you want to -use that instead, and `-override-input` will let you do that. +use that instead, and `--override-input` will let you do that. ### ... by testing against a haskell.nix project diff --git a/builder/default.nix b/builder/default.nix index 2e74fa24f..917aeeba3 100644 --- a/builder/default.nix +++ b/builder/default.nix @@ -18,7 +18,7 @@ let simplified = lib.mapAttrs (name: metas: builtins.map (m: m.version) metas) grouped; in simplified; - chap-package-components = package-name: package-version: + build-chap-package = package-name: package-version: let package-id = "${package-name}-${package-version}"; @@ -36,6 +36,10 @@ let "https://input-output-hk.github.io/cardano-haskell-packages" = CHaP; }; + # Note that we do not set tests or benchmarks to True, so we won't + # build them by default. This is the same as what happens on Hackage, + # for example, and they can't be depended on by downstream packages + # anyway. cabalProject = '' repository cardano-haskell-packages url: https://input-output-hk.github.io/cardano-haskell-packages @@ -58,7 +62,15 @@ let }; pkg = project.hsPkgs.${package-name}; components = haskellLib.getAllComponents pkg; - in lib.recurseIntoAttrs (builtins.listToAttrs (builtins.map (c: lib.nameValuePair c.pname c) components)); + in pkgs.releaseTools.aggregate { + name = package-id; + # Note that this does *not* include the derivations from 'checks' which + # actually run tests: CHaP will not check that your tests pass (neither + # does Hackage). + constituents = components; + }; in -# { foo = { X.Y.Z = ; P.Q.R = ; }; ... } -lib.recurseIntoAttrs (lib.mapAttrs (name: versions: lib.recurseIntoAttrs (lib.genAttrs versions (version: chap-package-components name version))) chap-package-attrs) +# { foo = { X.Y.Z = ; P.Q.R = ; }; ... } +lib.recurseIntoAttrs (lib.mapAttrs + (name: versions: lib.recurseIntoAttrs (lib.genAttrs versions (build-chap-package name))) + chap-package-attrs) From 4cf656b8a799842508a3713de139145e51c1ce51 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Fri, 17 Mar 2023 10:38:20 +0000 Subject: [PATCH 16/18] Refactor --- README.md | 6 +-- flake.nix | 54 ++++++++++++++------------ builder/default.nix => nix/builder.nix | 20 +--------- nix/chap-meta.nix | 29 ++++++++++++++ {builder => nix}/empty/almost/.gitkeep | 0 5 files changed, 63 insertions(+), 46 deletions(-) rename builder/default.nix => nix/builder.nix (69%) create mode 100644 nix/chap-meta.nix rename {builder => nix}/empty/almost/.gitkeep (100%) diff --git a/README.md b/README.md index ba645d668..7e4364cd8 100644 --- a/README.md +++ b/README.md @@ -268,10 +268,10 @@ You can build packages from CHaP using Nix like this: ``` nix build --override-input CHaP /home/user/cardano-haskell-packages/_repo - .#haskellPackages.x86_64-linux.ghc8107.plutus-core."1.1.0.0".plutus-core-exe-plc + .#haskellPackages.x86_64-linux.ghc8107.plutus-core."1.1.0.0" ``` -The final attribute name is `--`. +This will build all the components of that package version that CHaP cares about, namely libraries and executables (test suites and benchmarks are not built). We need to use `--override-input` because the CHaP flake relies on a built repository. By default it points to a built repository on the main CHaP `repo` branch. @@ -329,7 +329,7 @@ Along with requiring linear history, this ensures that package repository that w - Deploys the package repository to the `repo` branch, along with some static web content. - Builds a small set of packages using the newly built repository, to flush out any build issues. - We build with all the major GHC versions we expect to be in use. - - At the moment we don't build all the packages in the repository, only a fixed set. + - At the moment we don't build all the packages in the repository, only the latest versions of a fixed set. ## Creating a repository like CHaP diff --git a/flake.nix b/flake.nix index ea04c419e..f84e19e1a 100644 --- a/flake.nix +++ b/flake.nix @@ -34,16 +34,8 @@ }; outputs = { self, nixpkgs, flake-utils, foliage, haskell-nix, CHaP, iohk-nix, ... }: - let - compilers = [ "ghc8107" "ghc926" ]; - smokeTestPackages = [ - #"cardano-node" - #"cardano-cli" - #"cardano-api" - "plutus-core" - ]; # The foliage flake only works on this system, so we are stuck with it too - in flake-utils.lib.eachSystem [ "x86_64-linux" ] + flake-utils.lib.eachSystem [ "x86_64-linux" ] (system: let pkgs = import nixpkgs { @@ -55,6 +47,21 @@ ]; }; inherit (pkgs) lib; + + compilers = [ "ghc8107" "ghc926" ]; + + builder = import ./nix/builder.nix { inherit pkgs CHaP; }; + chap-meta = import ./nix/chap-meta.nix { inherit pkgs CHaP; }; + + smokeTestPackages = [ + #"cardano-node" + #"cardano-cli" + #"cardano-api" + "plutus-core" + ]; + # using intersectAttrs like this is a cheap way to throw away everything with keys not in + # smokeTestPackages + smokeTestPackageVersions = builtins.intersectAttrs (lib.genAttrs smokeTestPackages (pkg: {})) chap-meta.chap-package-latest-versions; in rec { devShells.default = with pkgs; mkShellNoCC { @@ -69,25 +76,23 @@ ]; }; - # { ghc8107 = { foo = { X.Y.Z = ; ... }; ...}; ... } + # { ghc8107 = { foo = { X.Y.Z = ; ... }; ...}; ... } haskellPackages = let - builder = import ./builder { inherit pkgs CHaP; }; + derivations = compiler: (lib.mapAttrs + (name: versions: (lib.genAttrs versions (version: builder compiler name version))) + chap-meta.chap-package-versions); in - lib.recurseIntoAttrs (lib.genAttrs compilers builder); - - # A nested tree of derivations containing all the smoke test packages for all the compiler versions - smokeTestDerivations = lib.genAttrs compilers (compiler: - let - # The latest version in the set (attrValues sorts by key). Remove the 'recurseForDerivations' here otherwise - # we get that! - latest = versionToValue: lib.last (lib.attrValues (builtins.removeAttrs versionToValue ["recurseForDerivations"])); - compilerPkgs = builtins.getAttr compiler haskellPackages; - smokeTestPkgs = lib.recurseIntoAttrs (lib.genAttrs smokeTestPackages (pkgname: latest (builtins.getAttr pkgname compilerPkgs))); - in smokeTestPkgs); + lib.genAttrs compilers derivations; # The standard checks: build all the smoke test packages - checks = flake-utils.lib.flattenTree smokeTestDerivations; + checks = + # We use recurseIntoAttrs so flattenTree will flatten it back out again. + let + derivations = compiler: lib.recurseIntoAttrs (lib.mapAttrs (name: version: builder compiler name version) smokeTestPackageVersions); + # A nested tree of derivations containing all the smoke test packages for all the compiler versions + perCompilerDerivations = lib.recurseIntoAttrs (lib.genAttrs compilers derivations); + in builtins.trace (builtins.toJSON smokeTestPackageVersions) (flake-utils.lib.flattenTree perCompilerDerivations); hydraJobs = checks; }); @@ -100,7 +105,8 @@ ]; extra-trusted-substituters = [ # If you have a nixbuild.net SSH key set up, you can pull builds from there - # by using '--extra-substituters ssh://eu.nixbuild.net' manually + # by using '--extra-substituters ssh://eu.nixbuild.net' manually, otherwise this + # does nothing "ssh://eu.nixbuild.net" ]; extra-trusted-public-keys = [ diff --git a/builder/default.nix b/nix/builder.nix similarity index 69% rename from builder/default.nix rename to nix/builder.nix index 917aeeba3..b9cfcc604 100644 --- a/builder/default.nix +++ b/nix/builder.nix @@ -4,20 +4,6 @@ let inherit (pkgs) lib; inherit (pkgs.haskell-nix) haskellLib; - # [ { name = "foo"; version = "X.Y.Z"; }, { name = "foo"; version = "P.Q.R"; } ] - chap-package-list = - builtins.map (p: { name = p.pkg-name; version = p.pkg-version; }) - (builtins.fromJSON (builtins.readFile "${CHaP}/foliage/packages.json")); - - # { "foo" : [ "X.Y.Z" "P.Q.R" ], ... } - chap-package-attrs = - let - # { "foo" = [{ name = "foo"; version = "X.Y.Z"; }, { name = "foo"; version = "P.Q.R"; }]; ... } - grouped = lib.groupBy (m: m.name) chap-package-list; - # { "foo" : [ "X.Y.Z" "P.Q.R" ], ... } - simplified = lib.mapAttrs (name: metas: builtins.map (m: m.version) metas) grouped; - in simplified; - build-chap-package = package-name: package-version: let package-id = "${package-name}-${package-version}"; @@ -69,8 +55,4 @@ let # does Hackage). constituents = components; }; -in -# { foo = { X.Y.Z = ; P.Q.R = ; }; ... } -lib.recurseIntoAttrs (lib.mapAttrs - (name: versions: lib.recurseIntoAttrs (lib.genAttrs versions (build-chap-package name))) - chap-package-attrs) +in build-chap-package diff --git a/nix/chap-meta.nix b/nix/chap-meta.nix new file mode 100644 index 000000000..42f8040b0 --- /dev/null +++ b/nix/chap-meta.nix @@ -0,0 +1,29 @@ +{ pkgs, CHaP }: +let + inherit (pkgs) lib; + inherit (pkgs.haskell-nix) haskellLib; + + + # [ { name = "foo"; version = "X.Y.Z"; }, { name = "foo"; version = "P.Q.R"; } ] + chap-package-list = + builtins.map (p: { name = p.pkg-name; version = p.pkg-version; }) + (builtins.fromJSON (builtins.readFile "${CHaP}/foliage/packages.json")); + + # { "foo" : [ "X.Y.Z" "P.Q.R" ], ... } + chap-package-versions = + let + # { "foo" = [{ name = "foo"; version = "X.Y.Z"; }, { name = "foo"; version = "P.Q.R"; }]; ... } + grouped = lib.groupBy (m: m.name) chap-package-list; + # { "foo" : [ "X.Y.Z" "P.Q.R" ], ... } + simplified = lib.mapAttrs (name: metas: builtins.map (m: m.version) metas) grouped; + in simplified; + + # { "foo" : "X.Y.Z", ... } + chap-package-latest-versions = + let latest = versions: lib.last (lib.naturalSort versions); + in lib.mapAttrs (name: versions: latest versions) chap-package-versions; + +in +{ + inherit chap-package-versions chap-package-latest-versions; +} diff --git a/builder/empty/almost/.gitkeep b/nix/empty/almost/.gitkeep similarity index 100% rename from builder/empty/almost/.gitkeep rename to nix/empty/almost/.gitkeep From d4aa7eddd7201f1bceb7dc994a84afaa294702d5 Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Fri, 17 Mar 2023 10:59:33 +0000 Subject: [PATCH 17/18] Bump hackage.nix and drop vector workaround --- flake.lock | 6 +++--- nix/builder.nix | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 09d291df8..e08053ffb 100644 --- a/flake.lock +++ b/flake.lock @@ -316,11 +316,11 @@ "hackage-nix": { "flake": false, "locked": { - "lastModified": 1678839912, - "narHash": "sha256-U13Gt95c4lAy+J1tAEInK4batuv4Az3q2Imar8V4qqA=", + "lastModified": 1679012701, + "narHash": "sha256-qWldqfEJEkqhQGEbx5yeZ+huc2muYj8WrMKx+uFqgMo=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "8d86d399acebee778bc0211008d0f1cea77b4307", + "rev": "a618dbd152ae5a8de7978ea5655096f7846cb880", "type": "github" }, "original": { diff --git a/nix/builder.nix b/nix/builder.nix index b9cfcc604..f4da1299f 100644 --- a/nix/builder.nix +++ b/nix/builder.nix @@ -8,6 +8,10 @@ let let package-id = "${package-name}-${package-version}"; + # Global config needed to build CHaP packages should go here. Obviously + # this should be kept to an absolute minimum, since that means config + # that every downstream project needs also. + # # No need to set index-state: # - haskell.nix will automatically use the latest known one for hackage # - we want the very latest state for CHaP so it includes anything from @@ -31,10 +35,6 @@ let url: https://input-output-hk.github.io/cardano-haskell-packages secure: True - -- Work around https://github.com/input-output-hk/nothunks/issues/17 - package nothunks - flags: +vector - extra-packages: ${package-id} ''; From ee0ca0bbaa6d6c43a52151f9651edb68cb2d56db Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Fri, 17 Mar 2023 11:12:38 +0000 Subject: [PATCH 18/18] Fix action version --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8dd435af5..4098ee20d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -113,7 +113,7 @@ jobs: extra_nix_config: accept-flake-config = true - name: Setup nixbuild.net - uses: nixbuild/nixbuild-action@biscuit + uses: nixbuild/nixbuild-action@v16 with: nixbuild_token: ${{secrets.NIXBUILD_TOKEN}}