workflows: Add new mkosi e2e flow

.github/workflows/caa_build_and_push.yaml

@@ -33,6 +33,11 @@ on:
         description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main.
         required: false
         type: string
+      runner:
+        default: 'ubuntu-24.04'
+        description: The runner to execute the workflow on. Defaults to 'ubuntu-24.04'.
+        required: false
+        type: string
 
 defaults:
   run:
@@ -41,7 +46,7 @@ defaults:
 jobs:
   build_push_job:
     name: build and push
-    runs-on: ubuntu-24.04
+    runs-on: ${{ inputs.runner }}
     strategy:
       fail-fast: false
       matrix:
@@ -65,6 +70,7 @@ jobs:
 
       - name: Read properties from versions.yaml
         run: |
+          sudo snap install yq
           go_version="$(yq '.tools.golang' versions.yaml)"
           [ -n "$go_version" ]
           echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV"

.github/workflows/caa_build_and_push_per_arch.yaml

@@ -58,7 +58,7 @@ jobs:
   build_push_job:
     name: build and push
     needs: [upload_tags]
-    runs-on: ubuntu-24.04
+    runs-on: ${{ matrix.type == 'dev-s390x' && 's390x' || 'ubuntu-24.04' }}
     strategy:
       fail-fast: false
       matrix:

.github/workflows/e2e_libvirt.yaml

@@ -7,6 +7,11 @@ name: (Callable) libvirt e2e tests
 on:
   workflow_call:
     inputs:
+      runner:
+        default: 'ubuntu-24.04'
+        description: The runner to execute the workflow on. Defaults to 'ubuntu-24.04'.
+        required: false
+        type: string
       podvm_image:
         required: true
         type: string
@@ -28,6 +33,12 @@ on:
         description: SecureComms configuration. Defaults to none.
         required: false
         type: string
+      oras:
+        description: Whether the podvm_image is oras published
+        default: false
+        required: false
+        type: boolean
+
 env:
   CLOUD_PROVIDER: libvirt
   DEBIAN_FRONTEND: noninteractive
@@ -38,7 +49,7 @@ defaults:
 
 jobs:
   test:
-    runs-on: ubuntu-24.04
+    runs-on: ${{ inputs.runner }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
@@ -69,10 +80,16 @@ jobs:
       - name: Setup docker
         if: ${{ runner.environment == 'self-hosted' }}
         run: |
+          sudo apt-get update -y
           sudo apt-get install -y docker.io
           sudo usermod -aG docker "$USER"
 
+      - uses: oras-project/setup-oras@v1
+        with:
+          version: ${{ env.ORAS_VERSION }}
+
       - name: Extract qcow2 from ${{ inputs.podvm_image }}
+        if: ${{ !inputs.oras }}
         run: |
            qcow2=$(echo ${{ inputs.podvm_image }} | sed -e "s#.*/\(.*\):.*#\1.qcow2#")
            ./hack/download-image.sh ${{ inputs.podvm_image }} . -o "${qcow2}" --clean-up
@@ -81,35 +98,58 @@ jobs:
            docker system prune -a -f
         working-directory: src/cloud-api-adaptor/podvm
 
-      - name: Get the install directory
-        if: ${{ inputs.install_directory_artifact != '' }}
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ inputs.install_directory_artifact }}
-          path: src/cloud-api-adaptor/install
+      - name: Use oras to get qcow2 from ${{ inputs.podvm_image }}
+        if: ${{ inputs.oras }}
+        run: |
+          oras pull ${{ inputs.podvm_image }}
+          tar xvJpf podvm.tar.xz
+          qcow2=$(find ./*.qcow2)
+          echo "PODVM_QCOW2=$(pwd)/${qcow2}" >> "$GITHUB_ENV"
+        working-directory: src/cloud-api-adaptor/podvm
 
       - name: Config Libvirt
         run: |
           export TEST_E2E_SECURE_COMMS="${{ inputs.secure_comms }}"
           ./libvirt/config_libvirt.sh
-          echo "CAA_IMAGE=\"${{ inputs.caa_image }}\"" >> libvirt.properties
           # For debugging
           cat libvirt.properties
 
-      - uses: oras-project/setup-oras@v1
-        with:
-          version: ${{ env.ORAS_VERSION }}
-
       - name: Install gh cli
         run: |
+          sudo apt update -y
           sudo apt install -y gh
 
+      - name: Double check that OVMF is installed
+        run: |
+          sudo apt update -y
+          sudo apt install -y ovmf
+
       - name: Install kustomize
         run: |
           command -v kustomize >/dev/null || \
           curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \
             sudo bash -s /usr/local/bin
 
+# For the legacy packer approach we don't want to use the default firmware, so comment it out
+      - name: Set blank firmware for packer libvirt tests
+        if: ${{ !inputs.oras }}
+        run: |
+          cd "install/overlays/libvirt"
+          sed -i 's/\(- LIBVIRT_EFI_FIRMWARE=.*\)/#\1/g' kustomization.yaml
+          # Print for debugging
+          echo "::group::Kustomization.yaml"
+          cat kustomization.yaml
+          echo "::endgroup::"
+
+      - name: Update kustomization configuration
+        run: |
+          cd "install/overlays/libvirt"
+          kustomize edit set image "cloud-api-adaptor=${{ inputs.caa_image }}"
+          # Print for debugging
+          echo "::group::libvirt kustomization"
+          cat kustomization.yaml
+          echo "::endgroup::"
+
       - name: Checkout KBS Repository
         run: |
           test/utils/checkout_kbs.sh
@@ -190,5 +230,5 @@ jobs:
         shell: bash {0}
 
       - name: Clean-up cluster
-        if: ${{ runner.environment == 'self-hosted' }}
+        if: ${{ always() && runner.environment == 'self-hosted' }}
         run: ./libvirt/kcli_cluster.sh delete

.github/workflows/e2e_run_all.yaml

@@ -66,6 +66,26 @@ jobs:
       git_ref: ${{ inputs.git_ref }}
     secrets: inherit
 
+  podvm_mkosi_amd64:
+    uses: ./.github/workflows/podvm_mkosi.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      image_tag: ${{ inputs.podvm_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+      arch: amd64
+      debug: true
+    secrets: inherit
+
+  podvm_mkosi_s390x:
+    uses: ./.github/workflows/podvm_mkosi.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      image_tag: ${{ inputs.podvm_image_tag }}
+      git_ref: ${{ inputs.git_ref }}
+      arch: s390x
+      debug: true
+    secrets: inherit
+
   # Build and push the cloud-api-adaptor image
   #
   # By using a reusable `workflow_call` workflow we are hitting two
@@ -97,7 +117,6 @@ jobs:
   # then please update the PROVIDERS list (space-separated names, e.g.,
   # "aws libvirt").
   prep_install:
-    needs: [image]
     runs-on: ubuntu-24.04
     outputs:
       matrix: ${{ steps.matrix.outputs.matrix }}
@@ -174,3 +193,92 @@ jobs:
       git_ref: ${{ inputs.git_ref }}
       secure_comms: ${{ matrix.secure_comms }}
     secrets: inherit
+
+  caa_image_amd64:
+    uses: ./.github/workflows/caa_build_and_push.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      dev_arches: 'linux/amd64'
+      release_arches: 'linux/amd64'
+      dev_tags: ${{ inputs.caa_image_tag }}-amd64-dev
+      release_tags: ${{ inputs.caa_image_tag }}-amd64
+      git_ref: ${{ inputs.git_ref }}
+    secrets: inherit
+
+  caa_image_s390x:
+    uses: ./.github/workflows/caa_build_and_push.yaml
+    with:
+      registry: ${{ inputs.registry }}
+      dev_arches: 'linux/s390x'
+      release_arches: 'linux/s390x'
+      dev_tags: ${{ inputs.caa_image_tag }}-s390x-dev
+      release_tags: ${{ inputs.caa_image_tag }}-s390x
+      git_ref: ${{ inputs.git_ref }}
+      runner: 's390x'
+    secrets: inherit
+
+  libvirt_e2e_arch_prep:
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.git_ref }}
+
+      - name: Rebase the code
+        if: github.event_name == 'pull_request_target'
+        working-directory: ./
+        run: |
+          ./hack/ci-helper.sh rebase-atop-of-the-latest-target-branch
+
+      - name: Define Test Matrix
+        id: matrix
+        run: |
+          echo "matrix=$(jq -c . < ./libvirt/libvirt_e2e_arch_matrix.json)" >> "$GITHUB_OUTPUT"
+
+  # Run libvirt amd64 e2e tests, based on the mkosi image, if pull request labeled 'test_e2e_libvirt'
+  libvirt_amd64:
+    name: E2E tests on libvirt for the amd64 architecture
+    if: |
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
+      contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_amd64')
+    needs: [podvm_mkosi_amd64, libvirt_e2e_arch_prep, caa_image_amd64]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.libvirt_e2e_arch_prep.outputs.matrix) }}
+    uses: ./.github/workflows/e2e_libvirt.yaml
+    with:
+      runner: ubuntu-24.04
+      caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-amd64-dev
+      podvm_image: ${{ needs.podvm_mkosi_amd64.outputs.qcow2_oras_image }}
+      install_directory_artifact: install_directory
+      git_ref: ${{ inputs.git_ref }}
+      oras: true
+    secrets: inherit
+
+  # Run libvirt s390x e2e tests, based on the mkosi image, if pull request labeled 'test_e2e_libvirt'
+  libvirt_s390x:
+    name: E2E tests on libvirt for the s390x architecture
+    if: |
+      github.event_name == 'schedule' ||
+      github.event_name == 'workflow_dispatch' ||
+      contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt') ||
+      contains(github.event.pull_request.labels.*.name, 'test_e2e_libvirt_s390x')
+    needs: [podvm_mkosi_s390x, libvirt_e2e_arch_prep, caa_image_s390x]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.libvirt_e2e_arch_prep.outputs.matrix) }}
+    uses: ./.github/workflows/e2e_libvirt.yaml
+    with:
+      runner: S390X
+      caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-s390x-dev
+      podvm_image: ${{ needs.podvm_mkosi_s390x.outputs.qcow2_oras_image }}
+      install_directory_artifact: install_directory
+      git_ref: ${{ inputs.git_ref }}
+      oras: true
+    secrets: inherit