Implement granular rust caching (#54582)

This fixes our caching for the docker builds as they were missing inputs
the other jobs had also enables caching the rust target cache which
improves build times when only changing our package and the lockfile
isn't invalidated.

Tested here https://github.com/vercel/next.js/actions/runs/5987764387

.github/workflows/build_and_deploy.yml

@@ -268,6 +268,13 @@ jobs:
         run: ${{ matrix.settings.setup }}
         if: ${{ matrix.settings.setup }}
 
+      - name: Cache on ${{ github.ref_name }}
+        uses: ijjk/[email protected]
+        with:
+          shared-key: build-${{ matrix.settings.target }}
+          save-if: 'true'
+          cache-provider: 'turbo'
+
       # we only need custom caching for docker builds
       # as they are on an older Node.js version and have
       # issues with turbo caching

.github/workflows/build_reusable.yml

@@ -49,6 +49,10 @@ on:
         required: false
         description: 'if test trace needs uploading'
         type: string
+      rustCacheKey:
+        required: false
+        description: 'rustCacheKey to cache shared target assets'
+        type: string
 
 env:
   NAPI_CLI_VERSION: 2.14.7
@@ -114,6 +118,14 @@ jobs:
 
       - run: corepack prepare --activate [email protected] && npm i -g "turbo@${TURBO_VERSION}" "@napi-rs/cli@${NAPI_CLI_VERSION}"
 
+      - name: Cache on ${{ github.ref_name }}
+        uses: ijjk/[email protected]
+        if: ${{ inputs.rustCacheKey }}
+        with:
+          shared-key: ${{ inputs.rustCacheKey }}-x86_64-unknown-linux-gnu
+          save-if: ${{ github.ref_name == 'canary' }}
+          cache-provider: 'turbo'
+
       # clean up any previous artifacts to avoid hitting disk space limits
       - run: git clean -xdf && rm -rf /tmp/next-repo-*; rm -rf /tmp/next-install-* /tmp/yarn-* /tmp/ncc-cache target
 

scripts/pull-turbo-cache.js

@@ -1,14 +1,35 @@
 #!/usr/bin/env node
 // @ts-check
 
-const { execSync } = require('child_process')
+const { spawn } = require('child_process')
 
 ;(async function () {
   const target = process.argv[process.argv.length - 1]
 
-  const turboResult = execSync(
-    `pnpm turbo run cache-build-native --dry=json -- ${target}`
-  ).toString()
+  let turboResult = ''
+
+  await new Promise((resolve, reject) => {
+    const child = spawn(
+      '/bin/bash',
+      ['-c', `pnpm turbo run cache-build-native --dry=json -- ${target}`],
+      {
+        stdio: 'pipe',
+      }
+    )
+
+    child.stdout.on('data', (data) => {
+      turboResult += data.toString()
+    })
+
+    child.on('exit', (code, signal) => {
+      if (code || signal) {
+        return reject(
+          new Error(`invalid exit code ${code} or signal ${signal}`)
+        )
+      }
+      resolve(0)
+    })
+  })
 
   const turboData = JSON.parse(turboResult)
 
@@ -21,10 +42,23 @@ const { execSync } = require('child_process')
 
   // pull cache if it was available
   if (task.cache.local || task.cache.remote) {
-    const pullResult = execSync(
-      `pnpm turbo run cache-build-native -- ${target}`
-    ).toString()
-    console.log(pullResult)
+    await new Promise((resolve, reject) => {
+      const child = spawn(
+        '/bin/bash',
+        ['-c', `pnpm turbo run cache-build-native -- ${target}`],
+        {
+          stdio: 'inherit',
+        }
+      )
+      child.on('exit', (code, signal) => {
+        if (code || signal) {
+          return reject(
+            new Error(`invalid exit code ${code} or signal ${signal}`)
+          )
+        }
+        resolve(0)
+      })
+    })
   } else {
     console.warn(`No turbo cache was available, continuing...`)
     console.warn(task)

turbo.json

@@ -74,6 +74,14 @@
       "outputs": ["crates/wasm/pkg/*"]
     },
     "cache-build-native": {
+      "inputs": [
+        "../../.cargo/**",
+        "../../packages/next-swc/crates/**",
+        "../../**/Cargo.toml",
+        "../../**/Cargo.lock",
+        "../../.github/workflows/build_and_deploy.yml",
+        "../../rust-toolchain"
+      ],
       "dependsOn": ["^cache-build-native"],
       "outputs": ["native/*.node"]
     },