From eaae2f57595df3b15c5380c31bfcaef7f3157b95 Mon Sep 17 00:00:00 2001 From: Kartik Joshi Date: Tue, 22 Aug 2023 16:39:52 +0530 Subject: [PATCH] azure: add workflow to build nightly podvm images Fixes: #1327 Add workflow to build podvm nightly image for the azure provider and share it with the azure e2e test workflow Co-authored-by: Kartik Joshi Signed-off-by: Magnus Kulke --- .github/workflows/azure-e2e-test.yml | 138 +++------------- .github/workflows/azure-podvm-image-build.yml | 151 ++++++++++++++++++ .../azure-podvm-image-nightly-build.yml | 27 ++++ ci-infra/azure/main.tf | 10 ++ ci-infra/azure/outputs.tf | 4 + 5 files changed, 211 insertions(+), 119 deletions(-) create mode 100644 .github/workflows/azure-podvm-image-build.yml create mode 100644 .github/workflows/azure-podvm-image-nightly-build.yml diff --git a/.github/workflows/azure-e2e-test.yml b/.github/workflows/azure-e2e-test.yml index 9b44d92d2..2f6d86c27 100644 --- a/.github/workflows/azure-e2e-test.yml +++ b/.github/workflows/azure-e2e-test.yml @@ -5,14 +5,7 @@ permissions: contents: read env: - PODVM_IMAGE_NAME: "peerpod-image-${{ github.run_id }}-${{ github.run_attempt }}" - SSH_USERNAME: "peerpod" - # VM size used for building image. - VM_SIZE: "Standard_D2as_v5" CLUSTER_NAME: "e2e-test-${{ github.run_id }}-${{ github.run_attempt }}" - KATA_AGENT_VERSION: "CC-0.7.0" - RUST_VERSION: "1.70.0" - GO_VERSION: "1.20" on: workflow_dispatch: @@ -22,114 +15,27 @@ on: description: prebuilt podvm image jobs: - build-podvm-image: + generate-podvm-image-version: + if: github.event.inputs.podvm-image-id == '' runs-on: ubuntu-latest - defaults: - run: - working-directory: cloud-api-adaptor/azure/image outputs: - pod-image-version: "${{ steps.generate_image_version.outputs.pod_image_version }}" - if: github.event.inputs.podvm-image-id == '' + image-version: "${{ steps.generate-image-version.outputs.image-version }}" steps: - - name: Clone cloud-api-adaptor repository - uses: actions/checkout@v3 - with: - path: cloud-api-adaptor - - - name: Generate version for pod vm image - id: generate_image_version + - name: Generate PodVM image version + id: generate-image-version run: | unique_version="$(date '+%m.%d.%H%M%S')${{ github.run_attempt }}" echo "Generated unique version for the image as: ${unique_version}" - echo "pod_image_version=${unique_version}" >> "$GITHUB_OUTPUT" - - - name: Set up Go environment - uses: actions/setup-go@v4 - with: - go-version: ${{ env.GO_VERSION }} - cache-dependency-path: cloud-api-adaptor/go.sum - - - name: Install build dependencies - run: sudo apt-get install -y musl-tools libdevmapper-dev libgpgme-dev - - - name: Build agent-protocol-forwarder - env: - GOPATH: /home/runner/go - run: make "$(realpath -m ../../podvm/files/usr/local/bin/agent-protocol-forwarder)" - - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ env.RUST_VERSION }} - target: x86_64-unknown-linux-musl - default: true - - - name: Set up kata-agent cache - id: kata-agent-cache - uses: actions/cache@v3 - with: - path: cloud-api-adaptor/podvm/files/usr/local/bin/kata-agent - key: kata-agent-${{ env.KATA_AGENT_VERSION }}_rust${{ env.RUST_VERSION }} - - - name: Clone kata-containers repository - if: steps.kata-agent-cache.outputs.cache-hit != 'true' - uses: actions/checkout@v3 - with: - repository: kata-containers/kata-containers - path: kata-containers - ref: ${{ env.KATA_AGENT_VERSION }} - - - name: Build kata-agent - env: - GOPATH: /home/runner/go - if: steps.kata-agent-cache.outputs.cache-hit != 'true' - run: make "$(realpath -m ../../podvm/files/usr/local/bin/kata-agent)" + echo "image-version=${unique_version}" >> "$GITHUB_OUTPUT" - - name: Set up pause cache - id: pause-cache - uses: actions/cache@v3 - with: - path: cloud-api-adaptor/podvm/files/pause_bundle - key: pause-${{ hashFiles('cloud-api-adaptor/podvm/Makefile.inc') }} - - - name: Build pause bundle - if: steps.pause-cache.outputs.cache-hit != 'true' - run: make "$(realpath -m ../../podvm/files/pause_bundle/rootfs/pause)" - - - name: Set up attestation-agent cache - id: aa-cache - uses: actions/cache@v3 - with: - path: cloud-api-adaptor/podvm/files/usr/local/bin/attestation-agent - key: aa-${{ hashFiles('cloud-api-adaptor/podvm/Makefile.inc') }} - - - name: Build attestation-agent - if: steps.aa-cache.outputs.cache-hit != 'true' - run: make "$(realpath -m ../../podvm/files/usr/local/bin/attestation-agent)" - - - uses: azure/login@v1 - name: 'Az CLI login' - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - - - name: Create podvm image - id: create-podvm-image - env: - PKR_VAR_subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - PKR_VAR_resource_group: ${{ secrets.AZURE_RESOURCE_GROUP }} - PKR_VAR_location: ${{ secrets.AZURE_REGION }} - PKR_VAR_az_image_name: ${{ env.PODVM_IMAGE_NAME }} - PKR_VAR_vm_size: ${{ env.VM_SIZE }} - PKR_VAR_ssh_username: ${{ env.SSH_USERNAME }} - PKR_VAR_az_gallery_name: ${{ secrets.AZURE_PODVM_GALLERY_NAME }} - PKR_VAR_az_gallery_image_name: ${{ secrets.AZURE_PODVM_IMAGE_DEF_NAME }} - PKR_VAR_az_gallery_image_version: ${{ steps.generate_image_version.outputs.pod_image_version }} - PKR_VAR_use_azure_cli_auth: "true" - PODVM_DISTRO: "ubuntu" - # Skip building of binaries - run: make image BINARIES= + build-podvm-image: + uses: confidential-containers/cloud-api-adaptor/.github/workflows/azure-podvm-image-build.yml@main + needs: + - generate-podvm-image-version + if: github.event.inputs.podvm-image-id == '' + secrets: inherit + with: + image-version: ${{ needs.generate-podvm-image-version.outputs.image-version }} build-caa-container-image: runs-on: ubuntu-latest @@ -187,7 +93,7 @@ jobs: - name: Create provisioner file env: TEST_E2E_CREATE_RG: "no" - PODVM_IMAGE_ID: "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ secrets.AZURE_RESOURCE_GROUP }}/providers/Microsoft.Compute/galleries/${{ secrets.AZURE_PODVM_GALLERY_NAME }}/images/${{ secrets.AZURE_PODVM_IMAGE_DEF_NAME }}/versions/${{ needs.build-podvm-image.outputs.pod-image-version }}" + AZURE_IMAGE_ID: "${{ github.event.inputs.podvm-image-id || needs.build-podvm-image.outputs.image-id }}" run: | cat << EOF > /tmp/provision_azure.properties AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" @@ -196,8 +102,7 @@ jobs: CLUSTER_NAME="$CLUSTER_NAME" LOCATION="${{ secrets.AZURE_REGION }}" SSH_KEY_ID="id_rsa.pub" - AZURE_IMAGE_ID="${{ github.event.inputs.podvm-image-id || env.PODVM_IMAGE_ID }}" - SSH_USERNAME="${{ env.SSH_USERNAME }}" + AZURE_IMAGE_ID="$AZURE_IMAGE_ID" IS_CI_MANAGED_CLUSTER="true" AZURE_CLI_AUTH="true" MANAGED_IDENTITY_NAME="${{ secrets.AZURE_MANAGED_IDENTITY_NAME}}" @@ -243,6 +148,7 @@ jobs: cleanup-resources: runs-on: ubuntu-latest needs: + - generate-podvm-image-version - build-podvm-image - build-caa-container-image - run-e2e-test @@ -260,19 +166,13 @@ jobs: - name: Remove podvm image if: github.event.inputs.podvm-image-id == '' - env: - PODVM_IMAGE_VERSION: "${{ needs.build-podvm-image.outputs.pod-image-version }}" run: | - # Delete the Pod VM image and its copy from the gallery. - az image delete \ - --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ - --name ${{ env.PODVM_IMAGE_NAME }} || true - + # Delete the Pod VM image from the gallery. az sig image-version delete \ --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ --gallery-name ${{ secrets.AZURE_PODVM_GALLERY_NAME }} \ --gallery-image-definition ${{ secrets.AZURE_PODVM_IMAGE_DEF_NAME }} \ - --gallery-image-version "${PODVM_IMAGE_VERSION}" || true + --gallery-image-version "${{ needs.generate-podvm-image-version.outputs.image-version }}" - name: Remove container image run: | diff --git a/.github/workflows/azure-podvm-image-build.yml b/.github/workflows/azure-podvm-image-build.yml new file mode 100644 index 000000000..5726abe45 --- /dev/null +++ b/.github/workflows/azure-podvm-image-build.yml @@ -0,0 +1,151 @@ +name: Azure PodVM image build + +on: + workflow_call: + inputs: + image-version: + type: string + required: true + outputs: + image-id: + description: "The PodVM image id" + value: ${{ jobs.build-podvm-image.outputs.image-id }} + + workflow_dispatch: + inputs: + image-version: + type: string + description: x.y.z + git-ref: + type: string + default: 'main' + description: tag, branch, sha + +permissions: + id-token: write + contents: read + +env: + AZURE_PODVM_IMAGE_DEF_NAME: "podvm_image0" + AZURE_PODVM_IMAGE_VERSION: "${{ inputs.image-version }}" + COMMUNITY_GALLERY_PREFIX: "/CommunityGalleries/cocopodvm-d0e4f35f-5530-4b9c-8596-112487cdea85" + GO_VERSION: "1.20" + KATA_AGENT_VERSION: "CC-0.7.0" + PODVM_IMAGE_NAME: "peerpod-image-${{ github.run_id }}-${{ github.run_attempt }}" + RUST_VERSION: "1.70.0" + SSH_USERNAME: "peerpod" + VM_SIZE: "Standard_D2as_v5" + +jobs: + build-podvm-image: + runs-on: ubuntu-latest + defaults: + run: + working-directory: cloud-api-adaptor/azure/image + outputs: + image-id: "${{ steps.create-image.outputs.image-id }}" + steps: + - name: Clone cloud-api-adaptor repository + uses: actions/checkout@v3 + with: + path: cloud-api-adaptor + ref: "${{ inputs.git-ref || 'main' }}" + + - name: Set up Go environment + uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + cache-dependency-path: cloud-api-adaptor/go.sum + + - name: Install build dependencies + run: sudo apt-get install -y musl-tools libdevmapper-dev libgpgme-dev + + - name: Build agent-protocol-forwarder + env: + GOPATH: /home/runner/go + run: make "$(realpath -m ../../podvm/files/usr/local/bin/agent-protocol-forwarder)" + + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ env.RUST_VERSION }} + target: x86_64-unknown-linux-musl + default: true + + - name: Set up kata-agent cache + id: kata-agent-cache + uses: actions/cache@v3 + with: + path: cloud-api-adaptor/podvm/files/usr/local/bin/kata-agent + key: kata-agent-${{ env.KATA_AGENT_VERSION }}_rust${{ env.RUST_VERSION }} + + - name: Clone kata-containers repository + if: steps.kata-agent-cache.outputs.cache-hit != 'true' + uses: actions/checkout@v3 + with: + repository: kata-containers/kata-containers + path: kata-containers + ref: ${{ env.KATA_AGENT_VERSION }} + + - name: Build kata-agent + env: + GOPATH: /home/runner/go + if: steps.kata-agent-cache.outputs.cache-hit != 'true' + run: make "$(realpath -m ../../podvm/files/usr/local/bin/kata-agent)" + + - name: Set up pause cache + id: pause-cache + uses: actions/cache@v3 + with: + path: cloud-api-adaptor/podvm/files/pause_bundle + key: pause-${{ hashFiles('cloud-api-adaptor/podvm/Makefile.inc') }} + + - name: Build pause bundle + if: steps.pause-cache.outputs.cache-hit != 'true' + run: make "$(realpath -m ../../podvm/files/pause_bundle/rootfs/pause)" + + - name: Set up attestation-agent cache + id: aa-cache + uses: actions/cache@v3 + with: + path: cloud-api-adaptor/podvm/files/usr/local/bin/attestation-agent + key: aa-${{ hashFiles('cloud-api-adaptor/podvm/Makefile.inc') }} + + - name: Build attestation-agent + if: steps.aa-cache.outputs.cache-hit != 'true' + run: make "$(realpath -m ../../podvm/files/usr/local/bin/attestation-agent)" + + - uses: azure/login@v1 + name: 'Az CLI login' + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + + - name: Create image + id: create-image + env: + IMAGE_ID: "${{ env.COMMUNITY_GALLERY_PREFIX }}/images/${{ env.AZURE_PODVM_IMAGE_DEF_NAME }}/versions/${{ env.AZURE_PODVM_IMAGE_VERSION }}" + PKR_VAR_subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + PKR_VAR_resource_group: ${{ secrets.AZURE_RESOURCE_GROUP }} + PKR_VAR_location: ${{ secrets.AZURE_REGION }} + PKR_VAR_az_image_name: ${{ env.PODVM_IMAGE_NAME }} + PKR_VAR_vm_size: ${{ env.VM_SIZE }} + PKR_VAR_ssh_username: ${{ env.SSH_USERNAME }} + PKR_VAR_az_gallery_name: ${{ secrets.AZURE_PODVM_GALLERY_NAME }} + PKR_VAR_az_gallery_image_name: ${{ env.AZURE_PODVM_IMAGE_DEF_NAME }} + PKR_VAR_az_gallery_image_version: ${{ env.AZURE_PODVM_IMAGE_VERSION }} + PKR_VAR_use_azure_cli_auth: "true" + PODVM_DISTRO: "ubuntu" + run: | + make image BINARIES= + echo "successfully built $IMAGE_ID" + echo "image-id=${IMAGE_ID}" >> "$GITHUB_OUTPUT" + + - name: Cleanup intermediate image + if: always() + run: | + # Delete intermediate image + az image delete \ + --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ + --name ${{ env.PODVM_IMAGE_NAME }} diff --git a/.github/workflows/azure-podvm-image-nightly-build.yml b/.github/workflows/azure-podvm-image-nightly-build.yml new file mode 100644 index 000000000..17cdd752e --- /dev/null +++ b/.github/workflows/azure-podvm-image-nightly-build.yml @@ -0,0 +1,27 @@ +name: azure podvm image nightly build + +on: + schedule: + # Run at 12:00 AM UTC + - cron: '0 0 * * *' + +jobs: + generate-image-version: + runs-on: ubuntu-latest + outputs: + image-version: "${{ steps.generate-image-version.outputs.image-version }}" + steps: + - name: Generate version for pod vm image + id: generate-image-version + run: | + nightly_version=$(date +'%Y.%m.%d') + echo "Generated nightly version for the image as: ${nightly_version}" + echo "image-version=${nightly_version}" >> "$GITHUB_OUTPUT" + + build-podvm-image: + needs: + - generate-image-version + uses: confidential-containers/cloud-api-adaptor/.github/workflows/azure-podvm-image-build.yml@main + secrets: inherit + with: + image-version: ${{ needs.generate-image-version.outputs.image-version }} diff --git a/ci-infra/azure/main.tf b/ci-infra/azure/main.tf index 6659740c5..82af0a9c4 100644 --- a/ci-infra/azure/main.tf +++ b/ci-infra/azure/main.tf @@ -72,6 +72,16 @@ resource "azurerm_shared_image_gallery" "podvm_image_gallery" { name = "${var.image_gallery}${var.ver}" resource_group_name = azurerm_resource_group.ci_rg.name location = azurerm_resource_group.ci_rg.location + + sharing { + permission = "Community" + community_gallery { + prefix = "cocopodvm" + eula = "https://raw.githubusercontent.com/confidential-containers/confidential-containers/main/LICENSE" + publisher_uri = "https://github.com/confidential-containers/confidential-containers" + publisher_email = "cocoatmsft@outlook.com" + } + } } resource "azurerm_shared_image" "podvm_image" { diff --git a/ci-infra/azure/outputs.tf b/ci-infra/azure/outputs.tf index bfccce40b..bd7528dd3 100644 --- a/ci-infra/azure/outputs.tf +++ b/ci-infra/azure/outputs.tf @@ -33,3 +33,7 @@ output "AZURE_PODVM_IMAGE_DEF_NAME" { output "AZURE_MANAGED_IDENTITY_NAME" { value = azurerm_user_assigned_identity.gh_action_user_identity.name } + +output "AZURE_COMMUNITY_GALLERY_NAME" { + value = azurerm_shared_image_gallery.podvm_image_gallery.sharing[0].community_gallery[0].name +}