WIP: Adds debug step

.github/workflows/build_rocks.yaml

@@ -0,0 +1,355 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+name: Build images
+
+on:
+  workflow_call:
+    inputs:
+      owner:
+        type: string
+        description: Registry owner to push the built images
+        default: ${{ github.repository_owner }}
+      registry:
+        type: string
+        description: Registry to push the built images
+        default: "ghcr.io"
+      runs-on:
+        type: string
+        description: Image runner for building the images
+        default: ubuntu-22.04
+      trivy-image-config:
+        type: string
+        description: Trivy YAML configuration for image testing that is checked in as part of the repo
+      working-directory:
+        type: string
+        description: The working directory for jobs
+        default: "./"
+      cache-action:
+        type: string
+        description: The cache action can either be "save", "restore", or "skip".
+        default: restore
+      multiarch-awareness:
+        type: boolean
+        description: Maintain the architecture labels on the container names
+        default: false
+      platform-labels:
+        type: string
+        description: |-
+          JSON mapping of rockcraft arches to gh runner labels
+          The key should be one of the platforms in rockcraft.yaml (amd64, arm64...)
+          the values should be a list of labels to use in the runs-on field of a gh action
+        default: '{}'
+      rockcraft-revisions:
+        type: string
+        description: |-
+          Pin the rockcraft snap revision -- per architecture
+
+          JSON mapping of rockcraft revisions
+          The key should be one of the platforms in rockcraft.yaml (amd64, arm64...)
+          The values should be a revision
+          If the key is missing for a discovered architecture, the revision will be ''
+        default: '{}'
+      arch-skipping-maximize-build-space:
+        type: string
+        description: |-
+          Some gh runners cannot use the maximize-build-spaces action
+          This config allows you skip that action on certain architectures.
+
+          JSON list of rockcraft arches to skip the maximize-build-space action
+          the values should be a rockcraft arch types which skip the maximize-build-space action
+
+          example)
+            arch-skipping-maximize-build-space: '["arm64"]'
+        default: '[]'
+    outputs:
+      images:
+        description: List of images built
+        value: ${{ jobs.get-rocks.outputs.images }}
+      rock-metas:
+        description: List of maps featuring the built {name, version, path, arch, image}
+        value: ${{ jobs.get-rocks.outputs.rock-metas }}
+      changed-rock-metas:
+        description: List of maps featuring the built that has changed {name, version, path, arch, image}
+        value: ${{ jobs.get-rocks.outputs.changed-rock-metas }}
+
+jobs:
+  get-rocks:
+    name: Get rocks
+    runs-on: ${{ inputs.runs-on }}
+    outputs:
+      rock-paths: ${{ steps.gen-rock-paths-and-images.outputs.rock-paths }}
+      images: "${{ steps.gen-rock-paths-and-images.outputs.images }}"
+      rock-metas: ${{ steps.gen-rock-paths-and-images.outputs.rock-metas }}
+      changed-rock-metas: ${{ steps.gen-rock-paths-and-images.outputs.changed-rock-metas }}
+    steps:
+      - name: Validate inputs
+        run: |
+          if [ "${{ inputs.cache-action }}" != "save" ] && [ "${{ inputs.cache-action }}" != "restore" ] && [ "${{ inputs.cache-action }}" != "skip" ]; then
+            echo "Invalid value for cache-action. It must be 'save', 'restore', or 'skip'"
+            exit 1
+          fi
+      - uses: actions/[email protected]
+      - uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          list-files: 'json'
+          filters: |
+            rocks:
+              - '**/rockcraft.yaml'
+      - name: Generate rock paths and images
+        id: gen-rock-paths-and-images
+        uses: actions/[email protected]
+        with:
+          script: |
+            const path = require('path')
+            const inputs = ${{ toJSON(inputs) }}
+            const workingDir = inputs['working-directory']
+            const multiarch = inputs['multiarch-awareness']
+            const rockcraftGlobber = await glob.create(
+                path.join(workingDir, '**/rockcraft.yaml')
+            )
+            const rockPaths = []
+            const images = []
+            const rockMetas = []
+            const changedMetas = []
+            const defaultArch = 'amd64'
+            const changes = ${{ toJSON(steps.changes) }}
+            const changesPaths = JSON.parse(changes['outputs']['rocks_files'])
+            const platformLabels = JSON.parse(inputs['platform-labels'])
+            const rockcraftRevisions = JSON.parse(inputs['rockcraft-revisions'])
+            const isPullRequest = ${{ github.event_name == 'pull_request' }}
+            core.info(`Multiarch Awareness is ${multiarch ? "on" : "off"}`)
+            for (const rockcraftFile of await rockcraftGlobber.glob()) {
+                const rockPath = path.relative('.', path.dirname(rockcraftFile)) || "./"
+                core.info(`found rockcraft.yaml in ${rockPath}`)
+                const fileHash = await glob.hashFiles(path.join(rockPath, '**'))
+                const [rockName, rockVersion] = (
+                    await exec.getExecOutput('yq', ['.name,.version', rockcraftFile])
+                ).stdout.trim().split("\n")
+                const platforms = (
+                    await exec.getExecOutput('yq', ['.platforms | keys', '-o=json', rockcraftFile])
+                ).stdout.trim()
+                if (multiarch && platforms) {
+                    const arches = JSON.parse(platforms)
+                    for (arch of arches) {
+                        const image = `${{inputs.registry}}/${{inputs.owner}}/${rockName}:${fileHash}-${arch}`
+                        core.info(`generate multi-arch image name: ${image}`)
+                        images.push(image)
+                        const meta = {
+                            name: rockName,
+                            version: rockVersion,
+                            path: rockPath,
+                            arch: arch,
+                            image: image,
+                            "rockcraft-revision": rockcraftRevisions[arch] || '',
+                            "runs-on-labels": platformLabels[arch] || [inputs["runs-on"]]
+                        }
+                        rockMetas.push(meta)
+                        const imageExists = (await exec.getExecOutput('docker', ['manifest', 'inspect', image], {ignoreReturnCode: true})).exitCode === 0
+                        if (isPullRequest && (changesPaths.includes(`${rockPath}/rockcraft.yaml`) || !imageExists)) {
+                            changedMetas.push(meta)
+                        } else if (!isPullRequest) {
+                            changedMetas.push(meta)
+                        }
+                    }
+                } else {
+                    const image = `${{inputs.registry}}/${{inputs.owner}}/${rockName}:${fileHash}`
+                    core.info(`generate image name: ${image}`)
+                    images.push(image)
+                    const meta = {
+                        name: rockName,
+                        version: rockVersion,
+                        path: rockPath,
+                        arch: defaultArch,
+                        image: image,
+                        "rockcraft-revision": rockcraftRevisions[defaultArch] || '',
+                        "runs-on-labels": platformLabels[defaultArch] || [inputs["runs-on"]]
+                    };
+                    rockMetas.push(meta)
+                    const imageExists = (await exec.getExecOutput('docker', ['manifest', 'inspect', image], {ignoreReturnCode: true})).exitCode === 0
+                    if (isPullRequest && (changesPaths.includes(`${rockPath}/rockcraft.yaml`) || !imageExists)) {
+                        changedMetas.push(meta)
+                    } else if (!isPullRequest) {
+                        changedMetas.push(meta)
+                    }
+                }
+                rockPaths.push(rockPath)
+            }
+            core.setOutput('rock-metas', JSON.stringify(rockMetas))
+            core.setOutput('changed-rock-metas', JSON.stringify(changedMetas))
+            core.setOutput('rock-paths', JSON.stringify(rockPaths))
+            core.setOutput('images', JSON.stringify(images))
+
+  build-rocks:
+    name: Build rock
+    needs: [get-rocks]
+    if: ${{ needs.get-rocks.outputs.changed-rock-metas != '[]' }}
+    strategy:
+      matrix:
+        rock: ${{ fromJSON(needs.get-rocks.outputs.changed-rock-metas) }}
+      fail-fast: false
+    runs-on: ${{ matrix.rock.runs-on-labels }}
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Ensure LXD storage pools path
+        run: |
+          sudo mkdir -p /var/snap/lxd/common/lxd/storage-pools
+      - name: Maximize build space
+        if: ${{ !contains(fromJson(inputs.arch-skipping-maximize-build-space), matrix.rock.arch) }}
+        uses: easimon/maximize-build-space@v10
+        with:
+          root-reserve-mb: 2048
+          temp-reserve-mb: 2048
+          overprovision-lvm: 'true'
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          build-mount-path: "/var/snap/lxd/common/lxd/storage-pools"
+          build-mount-path-ownership: "root:root"
+      - uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+      - name: Ensure tools on Runner
+        run: |
+          if ! which yq; then sudo snap install yq; fi
+          echo "path to yq=$(which yq)"
+
+      - name: Extract rock information
+        run: |
+          IMAGE_ARCH="${{ matrix.rock.arch }}"
+          IMAGE_NAME="${{ matrix.rock.name }}"
+          IMAGE_BASE=$(yq '.base' "${{ matrix.rock.path }}/rockcraft.yaml")
+          IMAGE_BUILD_BASE=$(yq '.["build-base"] // .base' "${{ matrix.rock.path }}/rockcraft.yaml")
+          IMAGE_REF=${{ matrix.rock.image }}
+          INODE_NUM=$(ls -id ${{ matrix.rock.path }} | cut -f 1 -d " ")
+          ROCKCRAFT_CONTAINER_NAME=rockcraft-$IMAGE_NAME-on-$IMAGE_ARCH-for-$IMAGE_ARCH-$INODE_NUM
+          echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV
+          echo "IMAGE_BASE=$IMAGE_BASE" >> $GITHUB_ENV
+          echo "IMAGE_BUILD_BASE=$IMAGE_BUILD_BASE" >> $GITHUB_ENV
+          echo "IMAGE_REF=$IMAGE_REF" >> $GITHUB_ENV
+          echo "IMAGE_ARCH=$IMAGE_ARCH" >> $GITHUB_ENV
+          echo "ROCKCRAFT_CONTAINER_NAME=$ROCKCRAFT_CONTAINER_NAME" >> $GITHUB_ENV
+      - name: Generate rockcraft cache key
+        run: |
+          ROCKCRAFT_PATH="${{ matrix.rock.path }}"
+          ROCKCRAFT_PATH="${ROCKCRAFT_PATH%/}"
+          ROCKCRAFT_CACHE_KEY_BASE="$ROCKCRAFT_PATH/rockcraft-cache?name=${{ env.IMAGE_NAME }}&base=${{ env.IMAGE_BUILD_BASE }}&build-base=${{ env.IMAGE_BUILD_BASE }}"
+          ROCK_CACHE_KEY_BASE="$ROCKCRAFT_PATH/${{ env.IMAGE_NAME }}.rock?filehash=${{ hashFiles(format('{0}/{1}', matrix.rock.path, '**')) }}"
+          if [ "${{ inputs.multiarch-awareness }}" == "true" ]; then
+            ROCKCRAFT_CACHE_KEY_BASE="${ROCKCRAFT_CACHE_KEY_BASE}&arch=${{ env.IMAGE_ARCH }}"
+            ROCK_CACHE_KEY_BASE="${ROCK_CACHE_KEY_BASE}&arch=${{ env.IMAGE_ARCH }}"
+          fi
+          echo "ROCKCRAFT_CACHE_KEY=$ROCKCRAFT_CACHE_KEY_BASE&date=$(date +%Y-%m-%d)" >> $GITHUB_ENV
+          echo 'ROCKCRAFT_CACHE_ALT_KEYS<<EOF' >> $GITHUB_ENV
+          for d in {1..2}
+            do echo "$ROCKCRAFT_CACHE_KEY_BASE&date=$(date -d"-$d days" +%Y-%m-%d)" >> $GITHUB_ENV
+          done
+          echo 'EOF' >> $GITHUB_ENV
+          echo "ROCK_CACHE_KEY=$ROCK_CACHE_KEY_BASE=$(date +%Y-%m-%d)" >> $GITHUB_ENV
+          echo 'ROCK_CACHE_ALT_KEYS<<EOF' >> $GITHUB_ENV
+          for d in {1..2}
+            do echo "$ROCK_CACHE_KEY_BASE&date=$(date -d"-$d days" +%Y-%m-%d)" >> $GITHUB_ENV
+          done
+          echo 'EOF' >> $GITHUB_ENV
+      - name: Restore rock cache
+        if: inputs.cache-action == 'restore'
+        uses: actions/cache/[email protected]
+        id: rock-cache
+        with:
+          path: ~/.rock-cache
+          key: ${{ env.ROCK_CACHE_KEY }}
+          restore-keys: ${{ env.ROCK_CACHE_ALT_KEYS }}
+      - name: Restore rockcraft container cache
+        if: steps.rock-cache.outputs.cache-hit != 'true' && inputs.cache-action == 'restore'
+        uses: actions/cache/[email protected]
+        id: rockcraft-cache
+        with:
+          path: ~/.rockcraft-cache/
+          key: ${{ env.ROCKCRAFT_CACHE_KEY }}
+          restore-keys: ${{ env.ROCKCRAFT_CACHE_ALT_KEYS }}
+      - name: Setup lxd
+        if: steps.rockcraft-cache.outputs.cache-hit == 'true'
+        run: |
+          sudo groupadd --force --system lxd
+          sudo usermod --append --groups lxd $USER
+          sudo snap refresh lxd --channel latest/stable
+          sudo lxd init --auto
+          sudo iptables -P FORWARD ACCEPT
+      - name: Import rockcraft container cache
+        if: steps.rockcraft-cache.outputs.cache-hit == 'true'
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          sudo lxc project create rockcraft -c features.images=false -c features.profiles=false
+          sudo lxc --project rockcraft import ~/.rockcraft-cache/${{ env.IMAGE_NAME }}.tar ${{ env.ROCKCRAFT_CONTAINER_NAME }}
+          find . -exec touch '{}' ';'
+      - name: Build rock
+        if: steps.rock-cache.outputs.cache-hit != 'true' || inputs.cache-action == 'save'
+        uses: canonical/craft-actions/rockcraft-pack@main
+        with:
+          path: ${{ matrix.rock.path }}
+          revision: ${{ matrix.rock.rockcraft-revision }}
+      - name: Generate rockcraft container cache
+        if: inputs.cache-action == 'save'
+        run: |
+          mkdir -p ~/.rockcraft-cache
+          mkdir -p ~/.rock-cache
+          touch ~/.rock-cache/.gitkeep
+          sudo lxc --project rockcraft export ${{ env.ROCKCRAFT_CONTAINER_NAME }} --compression none ~/.rockcraft-cache/${{ env.IMAGE_NAME }}.tar
+      - name: Delete rockcraft container cache
+        if: inputs.cache-action == 'save'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method DELETE \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/actions/caches?key=$(printf %s "${{ env.ROCKCRAFT_CACHE_KEY }}"|jq -sRr @uri) || :
+          for key in $(echo $ROCKCRAFT_CACHE_ALT_KEYS)
+            do gh api \
+              --method DELETE \
+              -H "Accept: application/vnd.github+json" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              /repos/${{ github.repository }}/actions/caches?key=$(printf %s "$key"|jq -sRr @uri) || :
+          done
+      - name: Save rockcraft container cache
+        if: inputs.cache-action == 'save'
+        uses: actions/cache/[email protected]
+        with:
+          path: ~/.rockcraft-cache/
+          key: ${{ env.ROCKCRAFT_CACHE_KEY }}
+      - name: Delete rock cache
+        if: inputs.cache-action == 'save'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method DELETE \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/actions/caches?key=$(printf %s "${{ env.ROCK_CACHE_KEY }}"|jq -sRr @uri) || :
+          for key in $(echo $ROCK_CACHE_ALT_KEYS)
+            do gh api \
+              --method DELETE \
+              -H "Accept: application/vnd.github+json" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              /repos/${{ github.repository }}/actions/caches?key=$(printf %s "$key"|jq -sRr @uri) || :
+          done
+      - name: Save rock cache
+        if: inputs.cache-action == 'save'
+        uses: actions/cache/[email protected]
+        with:
+          path: ~/.rock-cache
+          key: ${{ env.ROCK_CACHE_KEY }}
+      - name: Upload rock to ${{ inputs.registry }}
+        if: steps.rock-cache.outputs.cache-hit != 'true' || inputs.cache-action == 'save'
+        run: |
+          /snap/rockcraft/current/bin/skopeo --insecure-policy copy oci-archive:$(ls "${{ matrix.rock.path }}"/*.rock) docker://$IMAGE_REF --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}"
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 15

.github/workflows/pull_request.yaml

@@ -8,7 +8,7 @@ on:
 jobs:
   build-and-push-arch-specifics:
     name: Build Rocks and Push Arch Specific Images
-    uses: canonical/k8s-workflows/.github/workflows/build_rocks.yaml@main
+    uses: ./.github/workflows/build_rocks.yaml
     with:
       owner: ${{ github.repository_owner }}
       trivy-image-config: "trivy.yaml"