azure-e2e-test #327
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: azure-e2e-test | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| CLUSTER_NAME_TEMPLATE: "{0}-e2e-test-${{ github.run_id }}-${{ github.run_attempt }}" | |
| TEST_PROVISION_PATH_TEMPLATE: "${{ github.workspace }}/provision_azure${{ github.run_id }}_{0}.properties" | |
| CLOUD_PROVIDER: "azure" | |
| BUILTIN_CLOUD_PROVIDERS: "azure" | |
| TEST_E2E_CREATE_RG: "no" | |
| ACR_URL: "${{ vars.AZURE_ACR_URL }}" | |
| TEST_TAGS: "owner=github-actions,run=${{ github.run_id }}-${{ github.run_attempt }}" | |
| on: | |
| workflow_call: | |
| inputs: | |
| podvm-image-id: | |
| type: string | |
| description: prebuilt podvm image | |
| workflow_dispatch: | |
| inputs: | |
| podvm-image-id: | |
| type: string | |
| description: prebuilt podvm image | |
| caa-image: | |
| type: string | |
| description: prebuilt caa image | |
| jobs: | |
| build-caa-container-image: | |
| if: github.event.inputs.caa-image == '' | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| working-directory: src/cloud-api-adaptor | |
| outputs: | |
| caa-image: "${{ steps.build-container.outputs.caa-image }}" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: azure/login@v2 | |
| name: 'Az CLI login' | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build container image | |
| id: build-container | |
| env: | |
| ARCHES: "linux/amd64" | |
| RELEASE_BUILD: "false" | |
| run: | | |
| az acr login --name "$ACR_URL" | |
| # This builds image in the following format: | |
| # $ACR_URL/cloud-api-adaptor:dev-COMMIT | |
| make image "registry=${ACR_URL}" | |
| echo "caa-image=${ACR_URL}/cloud-api-adaptor:dev-${GITHUB_SHA}" >> "$GITHUB_OUTPUT" | |
| install-aks: | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| working-directory: src/cloud-api-adaptor | |
| needs: | |
| - build-caa-container-image | |
| # when none of required steps failed, skipped is ok | |
| if: always() && !failure() && !cancelled() | |
| strategy: | |
| matrix: | |
| parameters: | |
| - id: "tdx" | |
| machine_type: "Standard_DC2es_v5" | |
| jitter: 0 | |
| - id: "snp" | |
| machine_type: "Standard_DC2as_v5" | |
| jitter: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Extract go version number | |
| run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV" | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "${{ env.GO_VERSION }}" | |
| cache-dependency-path: "**/go.sum" | |
| - name: Set Provisioner Environment Variables | |
| run: | | |
| echo "TEST_PROVISION_FILE=${{ format(env.TEST_PROVISION_PATH_TEMPLATE, matrix.parameters.id) }}" >> "$GITHUB_ENV" | |
| echo "CLUSTER_NAME=${{ format(env.CLUSTER_NAME_TEMPLATE, matrix.parameters.id) }}" >> "$GITHUB_ENV" | |
| - name: Create provisioner file | |
| env: | |
| AZURE_IMAGE_ID: ${{ inputs.podvm-image-id }} | |
| CAA_IMAGE: "${{ github.event.inputs.caa-image || needs.build-caa-container-image.outputs.caa-image }}" | |
| AZURE_INSTANCE_SIZE: ${{ matrix.parameters.machine_type }} | |
| run: | | |
| echo "Provisioner file for ${{ matrix.parameters.id }}" | |
| cat << EOF > "$TEST_PROVISION_FILE" | |
| AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" | |
| AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" | |
| RESOURCE_GROUP_NAME="${{ secrets.AZURE_RESOURCE_GROUP }}" | |
| CLUSTER_NAME="${{ env.CLUSTER_NAME }}" | |
| LOCATION="${{ secrets.AZURE_REGION }}" | |
| SSH_KEY_ID="id_rsa.pub" | |
| AZURE_IMAGE_ID="$AZURE_IMAGE_ID" | |
| IS_CI_MANAGED_CLUSTER="true" | |
| MANAGED_IDENTITY_NAME="${{ secrets.AZURE_MANAGED_IDENTITY_NAME}}" | |
| CAA_IMAGE="${CAA_IMAGE}" | |
| AZURE_INSTANCE_SIZE="${AZURE_INSTANCE_SIZE}" | |
| TAGS="${{ env.TEST_TAGS }}" | |
| FEDERATED_CREDENTIAL_NAME="${{ env.CLUSTER_NAME }}" | |
| EOF | |
| cat "$TEST_PROVISION_FILE" | |
| # assert that no variable is unset | |
| ! grep -E '=x$|=""$' "$TEST_PROVISION_FILE" | |
| - name: Create public ssh key | |
| run: ssh-keygen -t rsa -b 4096 -f install/overlays/azure/id_rsa -N "" -C [email protected] | |
| - name: Save the configuration created here | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: | | |
| src/cloud-api-adaptor/install/overlays/azure/id_rsa.pub | |
| ${{ env.TEST_PROVISION_FILE }} | |
| name: e2e-configuration-${{ matrix.parameters.id }} | |
| - uses: azure/login@v2 | |
| name: 'Az CLI login' | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| # Install AKS cluster in parallel with the podvm image build. | |
| - name: Run provisioner | |
| working-directory: src/cloud-api-adaptor/test/tools | |
| run: | | |
| echo "Creating cluster for ${{ matrix.parameters.id }} machine" | |
| sleep ${{ matrix.parameters.jitter }} | |
| make caa-provisioner-cli | |
| ./caa-provisioner-cli -action=createcluster | |
| run-e2e-test: | |
| runs-on: ubuntu-24.04 | |
| defaults: | |
| run: | |
| working-directory: src/cloud-api-adaptor | |
| needs: | |
| - install-aks | |
| # when none of required steps failed, build-podvm-image can be skipped | |
| if: always() && !failure() && !cancelled() | |
| strategy: | |
| matrix: | |
| parameters: | |
| - id: "tdx" | |
| machine_type: "Standard_DC2es_v5" | |
| - id: "snp" | |
| machine_type: "Standard_DC2as_v5" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Extract version numbers | |
| run: | | |
| echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV" | |
| echo "ORAS_VERSION=$(yq -e '.tools.oras' versions.yaml)" >> "$GITHUB_ENV" | |
| - uses: oras-project/setup-oras@v1 | |
| with: | |
| version: ${{ env.ORAS_VERSION }} | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache-dependency-path: "**/go.sum" | |
| - name: Install cidr calculator | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y sipcalc | |
| - name: Install kustomize | |
| run: | | |
| command -v kustomize >/dev/null || \ | |
| curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | \ | |
| sudo bash -s /usr/local/bin | |
| - name: Set Provisioner Environment Variables | |
| run: | | |
| echo "TEST_PROVISION_FILE=${{ format(env.TEST_PROVISION_PATH_TEMPLATE, matrix.parameters.id) }}" >> "$GITHUB_ENV" | |
| echo "CLUSTER_NAME=${{ format(env.CLUSTER_NAME_TEMPLATE, matrix.parameters.id) }}" >> "$GITHUB_ENV" | |
| - name: Restore the configuration created before | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: e2e-configuration-${{ matrix.parameters.id }} | |
| - uses: azure/login@v2 | |
| name: 'Az CLI login' | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Create peerpod subnet | |
| run: | | |
| NODE_RG="$(az aks show -g ${{ secrets.AZURE_RESOURCE_GROUP }} -n "$CLUSTER_NAME" --query nodeResourceGroup -o tsv)" | |
| VNET_NAME="$(az network vnet list -g "$NODE_RG" --query '[].name' -o tsv)" | |
| NODE_CIDR="$(az network vnet show -n "$VNET_NAME" -g "$NODE_RG" --query "subnets[?name == 'aks-subnet'].addressPrefix" -o tsv)" | |
| MASK="${NODE_CIDR#*/}" | |
| PEERPOD_CIDR="$(sipcalc "$NODE_CIDR" -n 2 | grep ^Network | grep -v current | cut -d' ' -f2)/${MASK}" | |
| az network public-ip create -g "$NODE_RG" -n peerpod | |
| az network nat gateway create -g "$NODE_RG" -l ${{ secrets.AZURE_REGION }} --public-ip-addresses peerpod -n peerpod | |
| az network vnet subnet create -g "$NODE_RG" --vnet-name "$VNET_NAME" --nat-gateway peerpod --address-prefixes "$PEERPOD_CIDR" -n peerpod | |
| SUBNET_ID="$(az network vnet subnet show -g "$NODE_RG" --vnet-name "$VNET_NAME" -n peerpod --query id -o tsv)" | |
| echo "AZURE_SUBNET_ID=\"${SUBNET_ID}\"" >> "$TEST_PROVISION_FILE" | |
| - name: Checkout KBS Repository | |
| run: | | |
| test/utils/checkout_kbs.sh | |
| - name: Run e2e test | |
| env: | |
| TEST_PROVISION: "no" | |
| DEPLOY_KBS: "yes" | |
| CUSTOM_PCCS_URL: "https://global.acccache.azure.net/sgx/certification/v4" | |
| run: | | |
| # Since we install the cluster in parallel, we need to get the credentials here. | |
| echo "running e2e test for ${{ matrix.parameters.id }} machine" | |
| az aks get-credentials \ | |
| --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ | |
| --name "${CLUSTER_NAME}" | |
| make test-e2e RUN_TESTS="^Test\(CreateSimplePodAzure\|RemoteAttestation\)$" | |
| cleanup: | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - run-e2e-test | |
| if: always() | |
| strategy: | |
| matrix: | |
| parameters: | |
| - id: "tdx" | |
| machine_type: "Standard_DC2es_v5" | |
| jitter: 0 | |
| - id: "snp" | |
| machine_type: "Standard_DC2as_v5" | |
| jitter: 10 | |
| steps: | |
| - uses: azure/login@v2 | |
| name: 'Az CLI login' | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Delete coco namespace | |
| # We want to delete the coco namespace because CAA might still spawn resources | |
| # which prevents deletion of the AKS cluster | |
| run: | | |
| az aks get-credentials \ | |
| --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ | |
| --name "${{ format(env.CLUSTER_NAME_TEMPLATE, matrix.parameters.id) }}" || true | |
| namespace="confidential-containers-system" | |
| kubectl patch namespace "$namespace" -p '{"metadata":{"finalizers": null }}' || true | |
| kubectl delete namespace "$namespace" || true | |
| - name: Remove container image | |
| if: github.event.inputs.caa-image == '' | |
| run: | | |
| # Delete the CAA container image built for this run. | |
| suffix=".azurecr.io" | |
| registry_name="${ACR_URL%"${suffix}"}" | |
| az acr repository delete \ | |
| --name "${registry_name}" \ | |
| --image "${ACR_URL}/cloud-api-adaptor:dev-${GITHUB_SHA}" \ | |
| --yes || true | |
| - name: Remove dangling VMs | |
| # Remove any VMs that might have been left behind in failed test runs | |
| run: | | |
| vms=$(az resource list \ | |
| --tag owner=github-actions \ | |
| --tag run="${{ github.run_id }}-${{ github.run_attempt }}" \ | |
| -o tsv --query "[?type == 'Microsoft.Compute/virtualMachines'].name") | |
| for vm in $vms; do | |
| az vm delete -n "$vm" -g "${{ secrets.AZURE_RESOURCE_GROUP }}" --yes || true | |
| done | |
| - name: Remove dangling NICs | |
| # Remove any NICs that might have been left behind in failed test runs | |
| # NICs are reserved for 180s for VMs, even if they never launched | |
| run: | | |
| nics=$(az resource list \ | |
| --tag owner=github-actions \ | |
| --tag run="${{ github.run_id }}-${{ github.run_attempt }}" \ | |
| -o tsv --query "[?type == 'Microsoft.Network/networkInterfaces'].name") | |
| sleep 180 | |
| for nic in $nics; do | |
| az network nic delete -n "$nic" -g "${{ secrets.AZURE_RESOURCE_GROUP }}" || true | |
| done | |
| - name: Remove AKS cluster | |
| run: | | |
| # Delete the cluster even if it has been deleted already or does not exists. | |
| az aks delete \ | |
| --name "${{ format(env.CLUSTER_NAME_TEMPLATE, matrix.parameters.id) }}" \ | |
| --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ | |
| --no-wait \ | |
| --yes || true | |
| - name: Remove federated credential | |
| run: | | |
| sleep ${{ matrix.parameters.jitter }} | |
| az identity federated-credential delete \ | |
| --identity-name "${{ secrets.AZURE_MANAGED_IDENTITY_NAME }}" \ | |
| --name "${{ format(env.CLUSTER_NAME_TEMPLATE, matrix.parameters.id) }}" \ | |
| --resource-group "${{ secrets.AZURE_RESOURCE_GROUP }}" \ | |
| --yes || true |