Merge pull request #35 from DeterminateSystems/flakehub-cache

Introduce FlakeHub Cache

.github/workflows/cache-test.sh

@@ -1,14 +1,47 @@
-#!/bin/sh
+#! /usr/bin/env bash
 
 set -e
 set -ux
 
 seed=$(date)
 
+log="${MAGIC_NIX_CACHE_DAEMONDIR}/daemon.log"
+
+binary_cache=https://cache.flakehub.com
+
+# Check that the action initialized correctly.
+grep 'FlakeHub cache is enabled' "${log}"
+grep 'Using cache' "${log}"
+grep 'GitHub Action cache is enabled' "${log}"
+
+# Build something.
 outpath=$(nix-build .github/workflows/cache-tester.nix --argstr seed "$seed")
-nix copy --to 'http://127.0.0.1:37515' "$outpath"
+
+# Check that the path was enqueued to be pushed to the cache.
+grep "Enqueueing.*${outpath}" "${log}"
+
+# Wait until it has been pushed succesfully.
+found=
+for ((i = 0; i < 60; i++)); do
+    sleep 1
+    if grep "✅ $(basename "${outpath}")" "${log}"; then
+        found=1
+        break
+    fi
+done
+if [[ -z $found ]]; then
+    echo "FlakeHub push did not happen." >&2
+    exit 1
+fi
+
+# Check the FlakeHub binary cache to see if the path is really there.
+nix path-info --store "${binary_cache}" "${outpath}"
+
+# FIXME: remove this once the daemon also uploads to GHA automatically.
+nix copy --to 'http://127.0.0.1:37515' "${outpath}"
+
 rm ./result
-nix store delete "$outpath"
+nix store delete "${outpath}"
 if [ -f "$outpath" ]; then
     echo "$outpath still exists? can't test"
     exit 1

.github/workflows/ci.yml

@@ -16,9 +16,12 @@ jobs:
       - name: Record existing bundle hash
         run: |
           echo "BUNDLE_HASH=$(sha256sum <dist/index.js | sed 's/  -//')" >>$GITHUB_ENV
+      - name: Check shell scripts
+        run: |
+          nix develop --command shellcheck ./.github/workflows/cache-test.sh
       - name: Build action
         run: |
-          nix develop --command -- just build
+          nix develop --command just build
       - name: Check bundle consistency
         run: |
           NEW_BUNDLE_HASH=$(sha256sum <dist/index.js | sed 's/  -//')
@@ -35,11 +38,35 @@ jobs:
   run-x86_64-linux:
     name: Run x86_64 Linux
     runs-on: ubuntu-22.04
+    permissions:
+      id-token: "write"
+      contents: "read"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@main
+        with:
+          flakehub: true
+          extra-conf: |
+            narinfo-cache-negative-ttl = 0
+      - name: Cache the store
+        uses: ./
+      - name: Check the cache for liveness
+        run: |
+          .github/workflows/cache-test.sh
+
+  run-x86_64-darwin:
+    name: Run x86_64 Darwin
+    runs-on: macos-12
+    permissions:
+      id-token: "write"
+      contents: "read"
     steps:
       - uses: actions/checkout@v3
       - name: Install Nix
         uses: DeterminateSystems/nix-installer-action@main
         with:
+          flakehub: true
           extra-conf: |
             narinfo-cache-negative-ttl = 0
       - name: Cache the store
@@ -48,18 +75,23 @@ jobs:
         run: |
           .github/workflows/cache-test.sh
 
-  # run-x86_64-darwin:
-  #   name: Run x86_64 Darwin
-  #   runs-on: macos-12
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - name: Install Nix
-  #       uses: DeterminateSystems/nix-installer-action@main
-  #       with:
-  #         extra-conf: |
-  #           narinfo-cache-negative-ttl = 0
-  #     - name: Cache the store
-  #       uses: ./
-  #     - name: Check the cache for liveness
-  #       run: |
-  #         .github/workflows/cache-test.sh
+  run-aarch64-darwin:
+    name: Run aarch64 Darwin
+    concurrency: build-ARM64-macOS
+    runs-on: macos-latest-xlarge
+    permissions:
+      id-token: "write"
+      contents: "read"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Nix
+        uses: DeterminateSystems/nix-installer-action@main
+        with:
+          flakehub: true
+          extra-conf: |
+            narinfo-cache-negative-ttl = 0
+      - name: Cache the store
+        uses: ./
+      - name: Check the cache for liveness
+        run: |
+          .github/workflows/cache-test.sh

.github/workflows/publish.yml

@@ -0,0 +1,28 @@
+name: Publish
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  publish:
+    name: Publish
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          mkdir artifacts
+          git archive --format=tar.gz HEAD -o ./artifacts/universal.tar.gz
+
+      - uses: DeterminateSystems/push-artifact-ids@main
+        with:
+          s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE }}
+          bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET }}
+          directory: ./artifacts
+          ids_project_name: magic-nix-cache-action-priv
+          ids_binary_prefix: magic-nix-cache-action-priv

README.md

@@ -90,14 +90,20 @@ cat action.yml| nix run nixpkgs#yq-go -- '[[ "Parameter", "Description", "Requir
 | Parameter | Description | Required | Default |
 | - | - | - | - |
 | `diagnostic-endpoint` | Diagnostic endpoint url where diagnostics and performance data is sent. To disable set this to an empty string. |  | https://install.determinate.systems/magic-nix-cache/perf |
+| `flakehub-api-server` | The FlakeHub API server. |  | https://api.flakehub.com |
+| `flakehub-cache-server` | The FlakeHub binary cache server. |  | https://cache.flakehub.com |
+| `flakehub-flake-name` | The name of your flake on FlakeHub. Defaults to the current name of the GitHub repository it is running in. |  |  |
 | `listen` | The host and port to listen on. |  | 127.0.0.1:37515 |
 | `source-binary` | Run a version of the cache binary from somewhere already on disk. Conflicts with all other `source-*` options. |  |  |
-| `source-branch` | The branch of `magic-nix-cache` to use. Conflicts with all other `source-*` options. |  |  |
+| `source-branch` | The branch of `magic-nix-cache` to use. Conflicts with all other `source-*` options. |  | main |
 | `source-pr` | The PR of `magic-nix-cache` to use. Conflicts with all other `source-*` options. |  |  |
 | `source-revision` | The revision of `nix-magic-nix-cache` to use. Conflicts with all other `source-*` options. |  |  |
 | `source-tag` | The tag of `magic-nix-cache` to use. Conflicts with all other `source-*` options. |  |  |
 | `source-url` | A URL pointing to a `magic-nix-cache` binary. Overrides all other `source-*` options. |  |  |
-| `upstream-cache` | Your preferred [upstream cache](#upstream-cache). Store paths fetched from this store will not be cached in the [GitHub Actions Cache][gha-cache]. |  | https://cache.nixos.org |
+| `startup-notification-port` | The port magic-nix-cache uses for daemon startup notification. |  | 41239 |
+| `upstream-cache` | Your preferred upstream cache. Store paths in this store will not be cached in GitHub Actions' cache. |  | https://cache.nixos.org |
+| `use-flakehub` | Whether to upload build results to FlakeHub Cache. |  | true |
+| `use-gha-cache` | Whether to upload build results to the GitHub Actions cache. |  | true |
 
 [gha-cache]: https://docs.github.com/en/rest/actions/cache
 [detsys]: https://determinate.systems/

action.yml

@@ -4,6 +4,10 @@ branding:
   color: "purple"
 description: "Free, no-configuration Nix cache. Cut CI time by 50% or more by caching to GitHub Actions' cache."
 inputs:
+  use-gha-cache:
+    description: "Whether to upload build results to the GitHub Actions cache."
+    default: true
+    required: false
   listen:
     description: The host and port to listen on.
     default: 127.0.0.1:37515
@@ -30,6 +34,23 @@ inputs:
   diagnostic-endpoint:
     description: "Diagnostic endpoint url where diagnostics and performance data is sent. To disable set this to an empty string."
     default: "https://install.determinate.systems/magic-nix-cache/perf"
+  use-flakehub:
+    description: "Whether to upload build results to FlakeHub Cache."
+    default: true
+    required: false
+  flakehub-cache-server:
+    description: "The FlakeHub binary cache server."
+    default: "https://cache.flakehub.com"
+  flakehub-api-server:
+    description: "The FlakeHub API server."
+    default: "https://api.flakehub.com"
+  flakehub-flake-name:
+    description: "The name of your flake on FlakeHub. Defaults to the current name of the GitHub repository it is running in."
+    default: ${{ github.repository }}
+    required: false
+  startup-notification-port:
+    description: "The port magic-nix-cache uses for daemon startup notification."
+    default: 41239
 
 runs:
   using: "node20"