azure-e2e-test.yml

name: azure-e2e-test

permissions:
  id-token: write
  contents: read

env:
  CLUSTER_NAME: "e2e-test-${{ github.run_id }}-${{ github.run_attempt }}"
  TEST_PROVISION_FILE: "${{ github.workspace }}/provision_azure${{ github.run_id }}.properties"
  CLOUD_PROVIDER: "azure"
  BUILTIN_CLOUD_PROVIDERS: "azure"
  TEST_E2E_CREATE_RG: "no"
  ACR_URL: "${{ vars.AZURE_ACR_URL }}"

on:
  schedule:
  # Runs "at midnight every day" (see https://crontab.guru)
  # will base on default branch `main`
    - cron: '0 0 * * *'
  workflow_dispatch:
    inputs:
      podvm-image-id:
        type: string
        description: prebuilt podvm image
      caa-image:
        type: string
        description: prebuilt caa image

jobs:
  generate-podvm-image-version:
    if: github.event.inputs.podvm-image-id == ''
    runs-on: ubuntu-latest
    outputs:
      image-version: "${{ steps.generate-image-version.outputs.image-version }}"
    steps:
    - name: Generate PodVM image version
      id: generate-image-version
      run: |
        unique_version="$(date '+%m.%d.%H%M%S')${{ github.run_attempt }}"
        echo "Generated unique version for the image as: ${unique_version}"
        echo "image-version=${unique_version}" >> "$GITHUB_OUTPUT"

  build-podvm-image:
    uses: ./.github/workflows/azure-podvm-image-build.yml
    needs:
    - generate-podvm-image-version
    if: github.event.inputs.podvm-image-id == ''
    secrets: inherit
    with:
      image-version: ${{ needs.generate-podvm-image-version.outputs.image-version }}

  build-caa-container-image:
    if: github.event.inputs.caa-image == ''
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: src/cloud-api-adaptor
    outputs:
      caa-image: "${{ steps.build-container.outputs.caa-image }}"
    steps:
    - uses: actions/checkout@v3

    - name: Extract go version number
      run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV"

    - name: Set up Go environment
      uses: actions/setup-go@v4
      with:
        go-version: ${{ env.GO_VERSION }}
        cache-dependency-path: go.sum

    - uses: azure/login@v1
      name: 'Az CLI login'
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2

    - name: Install build dependencies
      run: |
        sudo apt-get update -y
        sudo apt-get install -y libvirt-dev

    - name: Build container image
      id: build-container
      env:
        ARCHES: "linux/amd64"
        RELEASE_BUILD: "false"
      run: |
        az acr login --name "$ACR_URL"
        # This builds image in the following format:
        # $ACR_URL/cloud-api-adaptor:dev-COMMIT
        make image "registry=${ACR_URL}"
        echo "caa-image=${ACR_URL}/cloud-api-adaptor:dev-${GITHUB_SHA}" >> "$GITHUB_OUTPUT"

  install-aks:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: src/cloud-api-adaptor
    needs:
    - generate-podvm-image-version
    - build-caa-container-image
    if: always() && !cancelled() && needs.build-caa-container-image.result != 'failure'
    steps:
    - uses: actions/checkout@v3

    - name: Extract go version number
      run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV"

    - name: Set up Go environment
      uses: actions/setup-go@v4
      with:
        go-version: ${{ env.GO_VERSION }}

    - name: Create provisioner file
      env:
        AZURE_IMAGE_ID: ${{ github.event.inputs.podvm-image-id || format('/CommunityGalleries/{0}/images/{1}/Versions/{2}', vars.AZURE_COMMUNITY_GALLERY_NAME, vars.AZURE_PODVM_IMAGE_DEF_NAME, needs.generate-podvm-image-version.outputs.image-version) }}
        CAA_IMAGE: "${{ github.event.inputs.caa-image || needs.build-caa-container-image.outputs.caa-image }}"
      run: |
        cat << EOF > "$TEST_PROVISION_FILE"
          AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
          AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}"
          RESOURCE_GROUP_NAME="${{ secrets.AZURE_RESOURCE_GROUP }}"
          CLUSTER_NAME="$CLUSTER_NAME"
          LOCATION="${{ secrets.AZURE_REGION }}"
          SSH_KEY_ID="id_rsa.pub"
          AZURE_IMAGE_ID="$AZURE_IMAGE_ID"
          IS_CI_MANAGED_CLUSTER="true"
          MANAGED_IDENTITY_NAME="${{ secrets.AZURE_MANAGED_IDENTITY_NAME}}"
          CAA_IMAGE="${CAA_IMAGE}"
        EOF
        cat "$TEST_PROVISION_FILE"
        # assert that no variable is unset
        ! grep -E '=x$|=""$' "$TEST_PROVISION_FILE"

    - name: Create public ssh key and secrets
      run: |
        # TODO: This is a temporary fix to create an empty file for env.
        touch install/overlays/azure/service-principal.env
        ssh-keygen -t rsa -b 4096 -f install/overlays/azure/id_rsa -N "" -C dev@coco.io

    - name: Save the configuration created here
      uses: actions/upload-artifact@v3
      with:
        path: |
          install/overlays/azure
          ${{ env.TEST_PROVISION_FILE }}
        name: e2e-configuration

    - uses: azure/login@v1
      name: 'Az CLI login'
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}

    # Install AKS cluster in parallel with the podvm image build.
    - name: Run provisioner
      working-directory: src/cloud-api-adaptor/test/tools
      run: |
        make caa-provisioner-cli
        ./caa-provisioner-cli -action=createcluster

  run-e2e-test:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: src/cloud-api-adaptor
    needs:
    - build-podvm-image
    - build-caa-container-image
    - install-aks
    - generate-podvm-image-version
    if: always() && !cancelled() && needs.build-podvm-image.result != 'failure'
    steps:
    - uses: actions/checkout@v3

    - name: Extract go version number
      run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV"

    - name: Set up Go environment
      uses: actions/setup-go@v4
      with:
        go-version: ${{ env.GO_VERSION }}

    - uses: azure/login@v1
      name: 'Az CLI login'
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}

    - name: Restore the configuration created before
      uses: actions/download-artifact@v3
      with:
        name: e2e-configuration

    - name: Add AKS Cluster Subnet ID to test provision file
      run: |
        NODE_RESOURCE_GROUP="$(az aks show -g ${{ secrets.AZURE_RESOURCE_GROUP }} -n "$CLUSTER_NAME" --query nodeResourceGroup -o tsv)"
        SUBNET_ID="$(az network vnet list -g "$NODE_RESOURCE_GROUP" --query '[0].subnets[0].id' -o tsv)"
        test -n "$SUBNET_ID"
        echo "AZURE_SUBNET_ID=\"${SUBNET_ID}\"" >> "$TEST_PROVISION_FILE"

    - name: Run e2e test
      env:
        TEST_PROVISION: "no"
      run: |
        # Since we install the cluster in parallel, we need to get the credentials here.
        az aks get-credentials \
          --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \
          --name "${CLUSTER_NAME}"
        make test-e2e

  cleanup-resources:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: src/cloud-api-adaptor
    needs:
    - generate-podvm-image-version
    - build-podvm-image
    - build-caa-container-image
    - run-e2e-test
    if: always()

    steps:
    - uses: actions/checkout@v3

    - name: Extract go version number
      run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV"

    - name: Set up Go environment
      uses: actions/setup-go@v4
      with:
        go-version: ${{ env.GO_VERSION }}

    - name: Restore the configuration created before
      uses: actions/download-artifact@v3
      with:
        name: e2e-configuration

    - uses: azure/login@v1
      name: 'Az CLI login'
      with:
        client-id: ${{ secrets.AZURE_CLIENT_ID }}
        subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
        tenant-id: ${{ secrets.AZURE_TENANT_ID }}

    # Clean up step, run regardless of the failure state.
    - name: Run deprovisioner
      working-directory: src/cloud-api-adaptor/test/tools
      run: |
        make caa-provisioner-cli
        # Ignore the error if the deprovision fails.
        ./caa-provisioner-cli -action=deprovision || true

    - name: Remove podvm image
      if: github.event.inputs.podvm-image-id == ''
      run: |
        # Delete the Pod VM image from the gallery.
        az sig image-version delete \
          --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \
          --gallery-name ${{ secrets.AZURE_PODVM_GALLERY_NAME }} \
          --gallery-image-definition ${{ vars.AZURE_PODVM_IMAGE_DEF_NAME }} \
          --gallery-image-version "${{ needs.generate-podvm-image-version.outputs.image-version }}" || true

    - name: Remove container image
      if: github.event.inputs.caa-image == ''
      run: |
        # Delete the CAA container image built for this run.
        suffix=".azurecr.io"
        registry_name="${ACR_URL%"${suffix}"}"

        az acr repository delete \
          --name "${registry_name}" \
          --image "${ACR_URL}/cloud-api-adaptor:dev-${GITHUB_SHA}" \
          --yes || true

    - name: Remove AKS cluster
      run: |
        # Delete the cluster even if it has been deleted already or does not exists.
        az aks delete \
          --name "${CLUSTER_NAME}" \
          --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \
          --no-wait \
          --yes || true