CI attempt 2 (#156)

* CI attempt 2

* Revise cardano-crypto-class

* Don't check out builder any more

* Small fixes for the builder

* later compiler

* Try this

* More new compiler

* Try multiline again

* Add some extra config for IFD

* Tweaks

* Add nixbuild.net as a substituter

* Major flake restructuring

* Just plutus-core for now

* Doc

* Comments

* Refactor

* Bump hackage.nix and drop vector workaround

* Fix action version

.github/workflows/CI.yml

@@ -20,25 +20,23 @@ jobs:
           fetch-depth: 0 # the check script below needs the whole history
 
       - name: Setup Nix
-        uses: cachix/install-nix-action@v17
+        uses: cachix/install-nix-action@v20
         with:
           nix_path: nixpkgs=channel:nixos-unstable
-          extra_nix_config: |
-            accept-flake-config = true
+          extra_nix_config: accept-flake-config = true
 
       - name: Run checks
         run: nix develop -c ./scripts/check.sh
 
-  build:
+  build-repo:
     runs-on: ubuntu-latest
 
     steps:
       - name: Setup Nix
-        uses: cachix/install-nix-action@v17
+        uses: cachix/install-nix-action@v20
         with:
           nix_path: nixpkgs=channel:nixos-unstable
-          extra_nix_config: |
-            accept-flake-config = true
+          extra_nix_config: accept-flake-config = true
 
       - name: Checkout main 
         uses: actions/checkout@v3
@@ -64,8 +62,10 @@ jobs:
           fi
 
       - name: Build repository (main)
+        # We don't need the metadata here, since we're just
+        # using this to compare the generated index
         run: |
-          nix develop -c foliage build -j 0
+          nix develop -c foliage build -j 0 
           mv _repo _repo-main
 
       - name: Checkout tip commit
@@ -84,7 +84,7 @@ jobs:
 
       # Do this before the check, useful to have the artifact in case the 
       # check fails!
-      - name: Upload artifact
+      - name: Upload built repository
         uses: actions/upload-artifact@v3
         with:
           name: built-repo
@@ -100,11 +100,52 @@ jobs:
           echo "then you may need to update the timestamps in your new packages to be newer than those in main."
           ./scripts/check-archive-extension.sh _repo-main/01-index.tar _repo/01-index.tar
 
+  build-packages:
+    runs-on: ubuntu-latest
+    needs:
+      - build-repo
+
+    steps:
+      - name: Install Nix
+        uses: cachix/install-nix-action@v20
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+          extra_nix_config: accept-flake-config = true
+
+      - name: Setup nixbuild.net
+        uses: nixbuild/nixbuild-action@v16
+        with:
+          nixbuild_token: ${{secrets.NIXBUILD_TOKEN}}
+
+      - uses: actions/checkout@v3
+
+      - name: Download built repository
+        uses: actions/download-artifact@v3
+        with:
+          name: built-repo
+          path: _repo
+
+      - name: Build checks
+        # The > is the "YAML folded string" marker, which replaces 
+        # newlines with spaces, since the usual bash idiom of \ 
+        # doesn't work for some reason
+        # 
+        # See https://github.com/nixbuild/feedback/issues/14 for
+        # why some of these options are here
+        run: >
+          nix flake check 
+          -L 
+          --override-input CHaP path:_repo 
+          --eval-store auto 
+          --store ssh-ng://eu.nixbuild.net 
+          --max-jobs 2
+          --builders ""
+
   deploy-check:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     needs: 
-      - build
+      - build-repo
 
     steps:
       - uses: actions/checkout@v3
@@ -135,7 +176,7 @@ jobs:
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     needs: 
       - check
-      - build
+      - build-repo
       - deploy-check
 
     concurrency:

README.md

@@ -220,7 +220,7 @@ Ideally, include the conditions under which we can deprecate it, e.g. "can depre
 ## How to test changes to CHaP 
 
 Sometimes it is useful to test in advance how a new package or a cabal file
-revision affects things
+revision affects things.
 
 The first steps are always the same, you need a built version of your modified 
 CHaP locally:
@@ -230,7 +230,7 @@ CHaP locally:
 For the rest of this section we will assume the built repository is in 
 `/home/user/cardano-haskell-packages/_repo`.
 
-### ... in isolation
+### ... by building packages with `cabal`
 
 You can test a locally built CHaP with a small test project consisting of just a
 `cabal.project` file:
@@ -259,9 +259,26 @@ Then you can build whatever package version you want with `cabal`:
 $ cabal build cardano-prelude
 ```
 
-You can troubleshoot a failed build plan using the cabal flags `--constraint`, `--allow-newer- and `--allow-older`. Once you have obtained a working build plan, you should revise you cabal file with appropriate constraints.
+You can troubleshoot a failed build plan using the cabal flags `--constraint`, `--allow-newer` and `--allow-older`. Once you have obtained a working build plan, you should revise you cabal file with appropriate constraints.
 
-### ... against haskell.nix projects
+### ... by building packages with Nix
+
+You can build packages from CHaP using Nix like this:
+
+```
+nix build 
+  --override-input CHaP /home/user/cardano-haskell-packages/_repo
+  .#haskellPackages.x86_64-linux.ghc8107.plutus-core."1.1.0.0"
+```
+
+This will build all the components of that package version that CHaP cares about, namely libraries and executables (test suites and benchmarks are not built).
+
+We need to use `--override-input` because the CHaP flake relies on a built repository. 
+By default it points to a built repository on the main CHaP `repo` branch. 
+But if you have just produced your own built repository (see above) then you want to
+use that instead, and `--override-input` will let you do that.
+
+### ... by testing against a haskell.nix project
 
 If you want to test a locally built CHaP against a project that uses CHaP 
 via haskell.nix, you can build the project while overriding CHaP
@@ -310,6 +327,9 @@ The CI for CHaP does the following things:
 Along with requiring linear history, this ensures that package repository that we build is always an extension of the previous one.
 - Builds the package repository from the metadata using `foliage`.
 - Deploys the package repository to the `repo` branch, along with some static web content.
+- Builds a small set of packages using the newly built repository, to flush out any build issues.
+    - We build with all the major GHC versions we expect to be in use.
+    - At the moment we don't build all the packages in the repository, only the latest versions of a fixed set.
 
 ## Creating a repository like CHaP
 

_sources/cardano-crypto-class/2.0.0.1/meta.toml

@@ -1,3 +1,7 @@
 timestamp = 2022-10-17T00:00:00Z
 github = { repo = "input-output-hk/cardano-base", rev = "4ef911eb7a4d628c938312ff4c92e7c0bb8c83a1" }
 subdir = 'cardano-crypto-class'
+
+[[revisions]]
+number = 1
+timestamp = 2023-03-15T16:30:48Z

_sources/cardano-crypto-class/2.0.0.1/revisions/1.cabal

@@ -0,0 +1,139 @@
+cabal-version:      2.2
+name:               cardano-crypto-class
+version:            2.0.0.1
+synopsis:
+  Type classes abstracting over cryptography primitives for Cardano
+
+description:
+  Type classes abstracting over cryptography primitives for Cardano
+
+license:            Apache-2.0
+license-files:
+  LICENSE
+  NOTICE
+
+author:             IOHK
+maintainer:         [email protected]
+copyright:          2019-2021 IOHK
+category:           Currency
+build-type:         Simple
+extra-source-files: README.md
+
+flag development
+  description: Disable `-Werror`
+  default:     False
+  manual:      True
+
+flag secp256k1-support
+    description: Enable support for functions from libsecp256k1. Requires
+                 a recent libsecp256k1 with support for Schnorr signatures.
+    default: True
+    manual: True
+
+common base                         { build-depends: base >= 4.14 && < 4.17 }
+
+common project-config
+  default-language: Haskell2010
+  ghc-options:
+    -Wall -Wcompat -Wincomplete-record-updates
+    -Wincomplete-uni-patterns -Wpartial-fields -Wredundant-constraints
+    -Wunused-packages
+
+library
+  import:            base, project-config
+  hs-source-dirs:    src
+  exposed-modules:
+    Cardano.Crypto.DSIGN
+    Cardano.Crypto.DSIGN.Class
+    Cardano.Crypto.DSIGN.Ed25519
+    Cardano.Crypto.DSIGN.Ed448
+    Cardano.Crypto.DSIGN.Mock
+    Cardano.Crypto.DSIGN.NeverUsed
+    Cardano.Crypto.Hash
+    Cardano.Crypto.Hash.Blake2b
+    Cardano.Crypto.Hash.Class
+    Cardano.Crypto.Hash.Keccak256
+    Cardano.Crypto.Hash.NeverUsed
+    Cardano.Crypto.Hash.SHA256
+    Cardano.Crypto.Hash.SHA3_256
+    Cardano.Crypto.Hash.Short
+    Cardano.Crypto.Init
+    Cardano.Crypto.KES
+    Cardano.Crypto.KES.Class
+    Cardano.Crypto.KES.CompactSingle
+    Cardano.Crypto.KES.CompactSum
+    Cardano.Crypto.KES.Mock
+    Cardano.Crypto.KES.NeverUsed
+    Cardano.Crypto.KES.Simple
+    Cardano.Crypto.KES.Single
+    Cardano.Crypto.KES.Sum
+    Cardano.Crypto.Libsodium
+    Cardano.Crypto.Libsodium.C
+    Cardano.Crypto.Libsodium.Constants
+    Cardano.Crypto.Libsodium.Hash
+    Cardano.Crypto.Libsodium.Init
+    Cardano.Crypto.Libsodium.Memory
+    Cardano.Crypto.Libsodium.Memory.Internal
+    Cardano.Crypto.Libsodium.MLockedBytes
+    Cardano.Crypto.Libsodium.MLockedBytes.Internal
+    Cardano.Crypto.Libsodium.UnsafeC
+    Cardano.Crypto.PinnedSizedBytes
+    Cardano.Crypto.Seed
+    Cardano.Crypto.Util
+    Cardano.Crypto.VRF
+    Cardano.Crypto.VRF.Class
+    Cardano.Crypto.VRF.Mock
+    Cardano.Crypto.VRF.NeverUsed
+    Cardano.Crypto.VRF.Simple
+    Cardano.Foreign
+
+  other-modules:
+    Cardano.Crypto.PackedBytes
+
+  build-depends:
+    , aeson
+    , base
+    , base16-bytestring  >=1
+    , bytestring
+    , cardano-binary
+    , cardano-strict-containers
+    , cryptonite
+    , deepseq
+    , heapwords
+    , memory
+    , nothunks
+    , primitive
+    , serialise
+    , template-haskell
+    , th-compat
+    , text
+    , transformers
+    , vector
+
+  if impl(ghc < 9.0.0)
+    build-depends:
+      integer-gmp
+
+  pkgconfig-depends: libsodium -any
+
+  if flag(secp256k1-support)
+    exposed-modules:
+      Cardano.Crypto.DSIGN.EcdsaSecp256k1
+      Cardano.Crypto.DSIGN.SchnorrSecp256k1
+      Cardano.Crypto.SECP256K1.Constants
+      Cardano.Crypto.SECP256K1.C
+    pkgconfig-depends: libsecp256k1 -any
+    cpp-options: -DSECP256K1_ENABLED
+
+test-suite test-memory-example
+  import:         base, project-config
+  type:           exitcode-stdio-1.0
+  hs-source-dirs: memory-example
+  main-is:        Main.hs
+  build-depends:
+    , base
+    , bytestring
+    , cardano-crypto-class
+
+  if (os(linux) || os(osx))
+    build-depends: unix