Skip to content

Commit

Permalink
feat: add container runtime archive configuration (#476)
Browse files Browse the repository at this point in the history
This change adds container runtime archive configuration as a tracked
dependency with weekly update automation against upstream nerdctl full
archives.

Signed-off-by: Austin Vazquez <[email protected]>
  • Loading branch information
austinvazquez authored Dec 27, 2024
1 parent 016abfe commit fb8861c
Show file tree
Hide file tree
Showing 10 changed files with 166 additions and 32 deletions.
1 change: 1 addition & 0 deletions .github/workflows/macos-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ on:
- main
paths:
- .github/workflows/macos-ci.yaml
- deps/container-runtime-full-archive.conf
- deps/full-os.conf
- deps/lima.conf
- e2e/**
Expand Down
82 changes: 74 additions & 8 deletions .github/workflows/update-dependencies.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,27 @@ on:
schedule:
- cron: '0 11 * * 2'
workflow_dispatch:
inputs:
nerdctl_tag_override:
type: string
description: The tag of nerdctl full archive to update
required: false
default: 'latest'

permissions:
# This is required for configure-aws-credentials to request an OIDC JWT ID token to access AWS resources later on.
# More info: https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings
id-token: write
contents: write
pull-requests: write
contents: read

jobs:
update-deps:
runs-on: ubuntu-latest

permissions:
# This is required for configure-aws-credentials to request an OIDC JWT ID token to access AWS resources later on.
# More info: https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect#adding-permissions-settings
id-token: write
contents: write
pull-requests: write

steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
Expand All @@ -25,9 +35,9 @@ jobs:
role-session-name: dependency-upload-session
aws-region: ${{ secrets.REGION }}

- name: Update Lima dependencies archive for Finch on macOS
run: bash bin/update-deps.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}
- name: Update Lima bundle archive for Finch on macOS
run: bash bin/update-lima-bundles.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}

- name: Update base OS for Finch on macOS
run: bash bin/update-os-image.sh -d ${{ secrets.DEPENDENCY_BUCKET_NAME }}

Expand All @@ -45,3 +55,59 @@ jobs:
signoff: true
title: 'build(deps): Bump finch dependencies'
branch: create-pull-request/bump-finch-dependencies

fetch-nerdctl-latest-release:
runs-on: ubuntu-latest

outputs:
tag: ${{ steps.latest-nerdctl-release.outputs.NERDCTL_TAG }}

steps:
- name: Checkout nerdctl
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: containerd/nerdctl
fetch-tags: true
- name: Fetch tag for latest release
id: nerdctl-latest-release
run: |
tag=$(git tag --sort=-version:refname | head -n 1)
if [[ -n ${{ github.event.inputs.nerdctl_tag_override }} ]]; then
tag=${{ github.event.inputs.nerdctl_tag_override }}
fi
echo "NERDCTL_TAG=$tag" >> $GITHUB_OUTPUT
update-container-runtime-full-archive:
runs-on: ubuntu-latest
needs: [fetch-nerdctl-latest-release]

permissions:
contents: write
pull-requests: write

env:
NERDCTL_TAG: ${{ needs.fetch-nerdctl-latest-release.outputs.tag }}

steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- name: Update container runtime dependencies
run: bash bin/update-container-runtime-deps.sh -t ${{ env.NERDCTL_TAG }}

- name: Create PR
uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
with:
token: ${{ secrets.GITHUB_TOKEN }}
signoff: true
branch: create-pull-request/update-container-runtime-full-archive-${{ env.NERDCTL_TAG }}
delete-branch: true
title: 'build(deps): update container runtime full archive to nerdctl-${{ env.NERDCTL_TAG }}'
add-paths: deps/container-runtime-full-archive.conf
body: |
Update the container runtime full archive to nerdctl ${{ env.NERDCTL_TAG }} for macOS and Window platforms.
See https://github.com/containerd/nerdctl/releases/tag/${{ env.NERDCTL_TAG }} for more details.
This PR created by [create-pull-request](https://github.com/peter-evans/create-pull-request) must be closed
and reopened manually to trigger automated checks.
3 changes: 0 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@ all: install.dependencies
install.dependencies: $(FINCH_DAEMON_OUTDIR)/finch-daemon

# Rootfs required for Windows, require full OS for Mac
FINCH_IMAGE_LOCATION ?=
FINCH_IMAGE_DIGEST ?=
FINCH_VM_TYPE ?=
BUILD_OS ?= $(OS)
ifeq ($(BUILD_OS), Windows_NT)
include Makefile.windows
Expand Down
22 changes: 16 additions & 6 deletions Makefile.darwin
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,23 @@ SOCKET_VMNET_TEMP_PREFIX ?= $(OUTDIR)/dependencies/lima-socket_vmnet/opt/finch

include $(CURDIR)/deps/full-os.conf
ifeq ($(ARCH),x86_64)
FINCH_OS_BASENAME ?= $(X86_64_ARTIFACT)
FINCH_OS_DIGEST ?= $(X86_64_512_DIGEST)
FINCH_OS_BASENAME := $(X86_64_ARTIFACT)
FINCH_OS_DIGEST := $(X86_64_512_DIGEST)
LIMA_ARCH = x86_64
else
FINCH_OS_BASENAME ?= $(AARCH64_ARTIFACT)
FINCH_OS_DIGEST ?= $(AARCH64_512_DIGEST)
FINCH_OS_BASENAME := $(AARCH64_ARTIFACT)
FINCH_OS_DIGEST := $(AARCH64_512_DIGEST)
LIMA_ARCH = aarch64
endif

FINCH_IMAGE_LOCATION := $(OS_OUTDIR)/$(FINCH_OS_BASENAME)
FINCH_IMAGE_DIGEST := "sha512:$(FINCH_OS_DIGEST)"
FINCH_IMAGE_LOCATION ?= $(OS_OUTDIR)/$(FINCH_OS_BASENAME)
FINCH_IMAGE_DIGEST ?= "sha512:$(FINCH_OS_DIGEST)"

include $(CURDIR)/deps/container-runtime-full-archive.conf
CONTAINER_RUNTIME_ARCHIVE_AARCH64_LOCATION ?= "$(ARTIFACT_BASE_URL)/$(AARCH64_ARTIFACT)"
CONTAINER_RUNTIME_ARCHIVE_AARCH64_DIGEST ?= "sha256:$(AARCH64_256_DIGEST)"
CONTAINER_RUNTIME_ARCHIVE_X86_64_LOCATION ?= "$(ARTIFACT_BASE_URL)/$(X86_64_ARTIFACT)"
CONTAINER_RUNTIME_ARCHIVE_X86_64_DIGEST ?= "sha256:$(X86_64_256_DIGEST)"

# Virtualization framework is the default virtual machine type on Finch on macOS
# This is only used for testing of Finch core bundles.
Expand Down Expand Up @@ -55,6 +61,10 @@ $(LIMA_TEMPLATE_OUTDIR)/fedora.yaml: $(LIMA_TEMPLATE_OUTDIR)
sed -i.bak -e "s|<image_location>|$(FINCH_IMAGE_LOCATION)|g" [email protected]
sed -i.bak -e "s/<image_arch>/$(LIMA_ARCH)/g" [email protected]
sed -i.bak -e "s/<image_digest>/$(FINCH_IMAGE_DIGEST)/g" [email protected]
sed -i.bak -e "s|<container_runtime_archive_aarch64_location>|$(CONTAINER_RUNTIME_ARCHIVE_AARCH64_LOCATION)|g" [email protected]
sed -i.bak -e "s/<container_runtime_archive_aarch64_digest>/$(CONTAINER_RUNTIME_ARCHIVE_AARCH64_DIGEST)/g" [email protected]
sed -i.bak -e "s|<container_runtime_archive_x86_64_location>|$(CONTAINER_RUNTIME_ARCHIVE_X86_64_LOCATION)|g" [email protected]
sed -i.bak -e "s/<container_runtime_archive_x86_64_digest>/$(CONTAINER_RUNTIME_ARCHIVE_X86_64_DIGEST)/g" [email protected]

# Replace was successful, so cleanup .bak files
rm $(LIMA_TEMPLATE_OUTDIR)/*.yaml.template.bak
Expand Down
4 changes: 2 additions & 2 deletions Makefile.windows
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ OUTPUT_DIRECTORIES += $(ROOTFS_OUTPUT_DIR)

include $(CURDIR)/deps/rootfs.conf
ifeq ($(ARCH), x86_64)
FINCH_ROOTFS_BASENAME ?= $(X86_64_ARTIFACT)
FINCH_ROOTFS_DIGEST ?= $(X86_64_512_DIGEST)
FINCH_ROOTFS_BASENAME := $(X86_64_ARTIFACT)
FINCH_ROOTFS_DIGEST := $(X86_64_512_DIGEST)
else
$(error Finch on Windows ARM not supported)
endif
Expand Down
49 changes: 49 additions & 0 deletions bin/update-container-runtime-full-archive.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
#!/usr/bin/env bash

# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

# A script to update the container runtime full archive used for Finch on macOS and Windows.
#
# Usage: bash update-container-runtime-full-archive.sh -t <Git tag>

set -euxo pipefail

CURRENT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd -- "${CURRENT_DIR}/.." && pwd)"

while getopts t: flag
do
case "${flag}" in
t) tag=${OPTARG};;
*) echo "Error: unknown flag" && exit 1;;
esac
done
[[ -z "$tag" ]] && { echo "Error: Git tag not set"; exit 1; }

DEPENDENCY_DOWNLOAD_BASE_URL="https://github.com/containerd/nerdctl/releases/download"
dependency_download_url="${DEPENDENCY_DOWNLOAD_BASE_URL}/${tag}"

# Pull upstream's published release shasums and save for later artifact verification.
mkdir -p "${PROJECT_ROOT}/downloads"
downloaded_shasums="${PROJECT_ROOT}/downloads/nerdctl-${tag}.sha256sums"
curl -L --fail "${dependency_download_url}/SHA256SUMS" > "${downloaded_shasums}"

version=${tag#v}
aarch64_deps="nerdctl-full-${version}-linux-arm64.tar.gz"
aarch64_deps_shasum=$(grep "${aarch64_deps}" "${downloaded_shasums}" | cut -d ' ' -f 1)
amd64_deps="nerdctl-full-${version}-linux-amd64.tar.gz"
amd64_deps_shasum=$(grep "${amd64_deps}" "${downloaded_shasums}" | cut -d ' ' -f 1)

# Update archive file with latest artifacts and digests.
ARCHIVE_FILE="${PROJECT_ROOT}/deps/container-runtime-full-archive.conf"
truncate -s 0 "${ARCHIVE_FILE}"
{
echo "ARTIFACT_BASE_URL=${dependency_download_url}"
echo ""
echo "AARCH64_ARTIFACT=${aarch64_deps}"
echo "AARCH64_256_DIGEST=${aarch64_deps_shasum}"
echo ""
echo "X86_64_ARTIFACT=${amd64_deps}"
echo "X86_64_256_DIGEST=${amd64_deps_shasum}"
} >> "${ARCHIVE_FILE}"
2 changes: 1 addition & 1 deletion bin/update-deps.sh → bin/update-lima-bundles.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

# A script to update the lima dependency used for Finch on MacOS and Windows.
#
# Usage: bash update-deps.sh -d <S3 bucket>
# Usage: bash update-lima-bundles.sh -d <S3 bucket>

set -euxo pipefail

Expand Down
18 changes: 11 additions & 7 deletions deps/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,16 @@ supported so the ARM configuration is not required in

## Updating artifact configuration

Artifact configuration for the Lima bundle for Finch on macOS and the rootfs
for Finch on Windows is updated via the
Artifact configuration for the Finch on macOS virtual machine image,
Lima bundles, rootfs for Finch on Windows, and
container runtime full archive is updated via the
[update dependencies](../.github/workflows/update-dependencies.yaml)
GitHub Actions workflow. The workflow scans S3 for more up-to-date
versions of the required dependency and opens a pull request with
the configuration updates.
GitHub Actions workflow.

Artifact configuration for the Finch on macOS virtual machine image
is still manually updated.
For Finch on macOS virtual machine image, Lima bundles, and Finch on Windows rootfs,
the workflow scans S3 for more up-to-date versions of the required dependency and
opens a pull request with the configuration updates.

For Finch on macOS and Windows container runtime full archive, the workflow checks
upstream for new upstream nerdctl releases and opens a pull request with the configuration
updates.
7 changes: 7 additions & 0 deletions deps/container-runtime-full-archive.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
ARTIFACT_BASE_URL=https://github.com/containerd/nerdctl/releases/download/v1.7.7

AARCH64_ARTIFACT=nerdctl-full-1.7.7-linux-arm64.tar.gz
AARCH64_256_DIGEST=b161a20c0e41f9ad999e8411e23c58ece4b3e584ae90b4252b76a39eee4a0c31

X86_64_ARTIFACT=nerdctl-full-1.7.7-linux-amd64.tar.gz
X86_64_256_DIGEST=a731eac93e8e9dda1a0d76dc1606438deb0668ea7d6bd5c5af436353ed9f65c5
10 changes: 5 additions & 5 deletions lima-template/fedora.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -118,12 +118,12 @@ containerd:
# # Override containerd archive
# # 🟢 Builtin default: hard-coded URL with hard-coded digest (see the output of `limactl info | jq .defaultTemplate.containerd.archives`)
archives:
- location: "https://github.com/containerd/nerdctl/releases/download/v1.7.7/nerdctl-full-1.7.7-linux-amd64.tar.gz"
arch: "x86_64"
digest: "sha256:a731eac93e8e9dda1a0d76dc1606438deb0668ea7d6bd5c5af436353ed9f65c5"
- location: "https://github.com/containerd/nerdctl/releases/download/v1.7.7/nerdctl-full-1.7.7-linux-arm64.tar.gz"
- location: "<container_runtime_archive_aarch64_location>"
arch: "aarch64"
digest: "sha256:b161a20c0e41f9ad999e8411e23c58ece4b3e584ae90b4252b76a39eee4a0c31"
digest: "<container_runtime_archive_aarch64_digest>"
- location: "<container_runtime_archive_x86_64_location>"
arch: "x86_64"
digest: "<container_runtime_archive_x86_64_digest>"

# Provisioning scripts need to be idempotent because they might be called
# multiple times, e.g. when the host VM is being restarted.
Expand Down

0 comments on commit fb8861c

Please sign in to comment.