Skip to content

Commit

Permalink
podvm: use ORAS to pull cdh and asr and aa
Browse files Browse the repository at this point in the history
The artifacts are being retrieved from guest-component's ORAS now. Hence
the rust build infrastructure can be removed with this change.

The Rust build infra hasn't been fully removed yet, this should be done
in a follow-up PR (otherwise the e2e test suite will fail, since those
run on main).

There are some notable changes:
- guest-component exposed the TEE_PLATFORM param on its top level build
  script, which we use to pull the correct artifact. Since we don't
  build attestation-agent directly anymore the ATTESTER param has been
  removed from the project's build scripts
- in versions.yaml kata and guest-components have been moved from the
  "git" section to the "oci" section, however since the tag is dynamic,
  we also provide a "reference" field in those entries.
- bumped guest-components to a commit that is available as artifact in
  OCI

Signed-off-by: Magnus Kulke <[email protected]>
  • Loading branch information
mkulke committed Oct 22, 2024
1 parent ea8518a commit 4de6783
Show file tree
Hide file tree
Showing 21 changed files with 71 additions and 105 deletions.
3 changes: 0 additions & 3 deletions .github/workflows/azure-e2e-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -232,9 +232,6 @@ jobs:
SUBNET_ID="$(az network vnet subnet show -g "$NODE_RG" --vnet-name "$VNET_NAME" -n peerpod --query id -o tsv)"
echo "AZURE_SUBNET_ID=\"${SUBNET_ID}\"" >> "$TEST_PROVISION_FILE"
- name: Extract kbs reference
run: echo "KBS_VERSION=$(yq -e '.git.kbs.reference' versions.yaml)" >> "$GITHUB_ENV"

- name: Checkout KBS Repository
run: |
test/utils/checkout_kbs.sh
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/azure-podvm-image-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ jobs:
- name: Build binaries
run: |
make fedora-binaries-builder
ATTESTER="az-snp-vtpm-attester,az-tdx-vtpm-attester" make binaries
TEE_PLATFORM=az-cvm-vtpm make binaries
- name: Build image
run: make image
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/e2e_libvirt.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,10 @@ jobs:
sudo snap install yq
echo "KBS_REPO=$(yq -e '.git.kbs.url' versions.yaml)" >> "$GITHUB_ENV"
echo "KBS_VERSION=$(yq -e '.git.kbs.reference' versions.yaml)" >> "$GITHUB_ENV"
echo "RUST_VERSION=$(yq -e '.tools.rust' versions.yaml)" >> "$GITHUB_ENV"
go_version="$(yq '.tools.golang' versions.yaml)"
[ -n "$go_version" ]
echo "GO_VERSION=${go_version}" >> "$GITHUB_ENV"
echo "ORAS_VERSION=$(yq -e '.tools.oras' versions.yaml)" >> "$GITHUB_ENV"
- name: Setup Golang version ${{ env.GO_VERSION }}
uses: actions/setup-go@v5
Expand Down Expand Up @@ -96,6 +96,10 @@ jobs:
override: true
profile: minimal

- uses: oras-project/setup-oras@v1
with:
version: ${{ env.ORAS_VERSION }}

- name: Install gh cli
run: |
sudo apt install -y gh
Expand Down
2 changes: 1 addition & 1 deletion docs/Release-Process.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ version is picked) and [trustee releases](https://github.com/confidential-contai
`externals.coco-guest-components.version`, `externals.coco-trustee` and the `image-rs` crate in the agent's
[`Cargo.toml`](https://github.com/kata-containers/kata-containers/blob/main/src/agent/Cargo.toml).
- At this point it makes sense for us to stay in sync, by updating the guest-components and kbs that we use in peer pods,
by changing the `git.guest-components.reference` and `git.kbs.tag` values in [versions.yaml](../src/cloud-api-adaptor/versions.yaml).
by changing the `oci.guest-components.reference`, `oci.kbs.tag` and `oci.kbs-client.reference` values in [versions.yaml](../src/cloud-api-adaptor/versions.yaml).
We should also bump the kata agent to the latest commit
hash in our [version.yaml](../src/cloud-api-adaptor/versions.yaml) for testing.
1. Kata Containers [releases](https://github.com/kata-containers/kata-containers/releases)
Expand Down
4 changes: 1 addition & 3 deletions src/cloud-api-adaptor/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ BINARIES := cloud-api-adaptor agent-protocol-forwarder process-user-data
SOURCEDIRS := ./cmd ./pkg
PACKAGES := $(shell go list $(addsuffix /...,$(SOURCEDIRS)))
SOURCES := $(shell find $(SOURCEDIRS) -name '*.go' -print)
ATTESTER ?= none
# End-to-end tests overall run timeout.
TEST_E2E_TIMEOUT ?= 60m
# To run a sub-set of tests, set an unanchored regex matching the test's name as in go's test -run option.
Expand Down Expand Up @@ -182,7 +181,6 @@ podvm-builder:
--build-arg ACTIVATION_KEY=$(ACTIVATION_KEY) \
--build-arg ARCH=$(ARCH) \
--build-arg PROTOC_VERSION=$(PROTOC_VERSION) \
--build-arg RUST_VERSION=$(RUST_VERSION) \
--build-arg YQ_VERSION=$(YQ_VERSION) \
--build-arg YQ_CHECKSUM=${YQ_CHECKSUM_$(ARCH)} \
--build-arg YQ_ARCH=$(ARCH) \
Expand All @@ -197,7 +195,7 @@ podvm-binaries:
--build-arg BUILDER_IMG=$(PODVM_BUILDER_IMAGE) \
--build-arg PODVM_DISTRO=$(PODVM_DISTRO) \
--build-arg ARCH=$(ARCH) \
--build-arg ATTESTER=$(ATTESTER) \
--build-arg TEE_PLATFORM=$(TEE_PLATFORM) \
--build-arg PAUSE_REPO=$(PAUSE_REPO) \
--build-arg PAUSE_VERSION=$(PAUSE_VERSION) \
--build-arg PAUSE_BIN=$(PAUSE_BIN) \
Expand Down
13 changes: 7 additions & 6 deletions src/cloud-api-adaptor/Makefile.defaults
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ PAUSE_BIN ?= pause
# stored in versions.yaml
YQ_VERSION := v4.35.1
YQ_CHECKSUM := "sha256:bd695a6513f1196aeda17b174a15e9c351843fb1cef5f9be0af170f2dd744f08"
YQ_CHECKSUM_s390x:= "sha256:4e6324d08630e7df733894a11830412a43703682d65a76f1fc925aac08268a45"
YQ_CHECKSUM_s390x := "sha256:4e6324d08630e7df733894a11830412a43703682d65a76f1fc925aac08268a45"
# none,az-cvm-vtpm,tdx,se,cca
TEE_PLATFORM ?= none

VERSIONS_HASH := $(firstword $(shell sha256sum $(VERSIONS_SRC)))

Expand All @@ -30,18 +32,17 @@ rhel_s390x_IMAGE_CHECKSUM := $(call query,cloudimg.rhel.$(rhel_RELEASE).s390x.ch

COCO_OPERATOR_REF := $(or $(COCO_OPERATOR_REF),$(call query,git.coco-operator.reference))
COCO_OPERATOR_REPO := $(or $(COCO_OPERATOR_REPO),$(call query,git.coco-operator.url))
KATA_SRC := $(or $(KATA_SRC),$(call query,git.kata-containers.url))
KATA_SRC_REF := $(or $(KATA_SRC_REF),$(call query,git.kata-containers.reference))
KATA_REGISTRY := $(or $(KATA_REGISTRY), $(call query,oci.kata-containers.registry))
KATA_REF := $(or $(KATA_REF), $(call query,oci.kata-containers.reference))
GO_VERSION := $(or $(GO_VERSION),$(call query,tools.golang))
PROTOC_VERSION := $(or $(PROTOC_VERSION),$(call query,tools.protoc))
RUST_VERSION := $(or $(RUST_VERSION),$(call query,tools.rust))
ORAS_VERSION := $(or $(ORAS_VERSION),$(call query,tools.oras))
SKOPEO_REPO := $(or $(SKOPEO_REPO),$(call query,git.skopeo.url))
SKOPEO_VERSION := $(or $(SKOPEO_VERSION),$(call query,git.skopeo.reference))
UMOCI_REPO := $(or $(UMOCI_REPO),$(call query,git.umoci.url))
UMOCI_VERSION := $(or $(UMOCI_VERSION),$(call query,git.umoci.reference))
GUEST_COMPONENTS_REPO := $(or $(GUEST_COMPONENTS_REPO),$(call query,git.guest-components.url))
GUEST_COMPONENTS_VERSION := $(or $(GUEST_COMPONENTS_VERSION),$(call query,git.guest-components.reference))
GUEST_COMPONENTS_REGISTRY := $(or $(GUEST_COMPONENTS_REGISTRY), $(call query,oci.guest-components.registry))
GUEST_COMPONENTS_REF := $(or $(GUEST_COMPONENTS_REF),$(call query,oci.guest-components.reference))
PAUSE_REPO := $(or $(PAUSE_REPO),$(call query,oci.pause.registry))
PAUSE_VERSION := $(or $(PAUSE_VERSION),$(call query,oci.pause.tag))
PACKER_VERSION := $(or $(PACKER_VERSION),$(call query,tools.packer))
3 changes: 1 addition & 2 deletions src/cloud-api-adaptor/azure/build-image.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ sudo apt install \

- Install `yq` by following instructions [here](https://mikefarah.gitbook.io/yq/#install).
- Install Golang by following instructions [here](https://go.dev/doc/install).
- Install Rust (`cargo`) by following instructions [here](https://www.rust-lang.org/tools/install).
- Install packer by following [these instructions](https://learn.hashicorp.com/tutorials/packer/get-started-install-cli).

**Build**
Expand All @@ -110,7 +109,7 @@ export PKR_VAR_az_gallery_image_version="0.0.1"
export PKR_VAR_offer=0001-com-ubuntu-confidential-vm-jammy
export PKR_VAR_sku=22_04-lts-cvm

export ATTESTER="az-snp-vtpm-attester,az-tdx-vtpm-attester"
export TEE_PLATFORM="az-cvm-vtpm"
export LIBC=gnu
export CLOUD_PROVIDER=azure
PODVM_DISTRO=ubuntu make image
Expand Down
12 changes: 6 additions & 6 deletions src/cloud-api-adaptor/hack/verify-provenance.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@
set -euo pipefail

usage() {
echo "Usage: $0 "
echo " -a <oci-artifact w/ sha256 digest>"
echo " -d <expected git sha1 from which the artifact was built>"
echo " -r <repository on which the artifact was built>"
echo " [-g] (optional. fetch attestation using github api)"
exit 1
echo "Usage: $0 "
echo " -a <oci-artifact w/ sha256 digest>"
echo " -d <expected git sha1 from which the artifact was built>"
echo " -r <repository on which the artifact was built>"
echo " [-g] (optional. fetch attestation using github api)"
exit 1
}

oci_artifact=""
Expand Down
5 changes: 0 additions & 5 deletions src/cloud-api-adaptor/ibmcloud-powervs/image/prereq.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

# FIXME to pickup these values from versions.yaml
GO_VERSION="1.22.7"
RUST_VERSION="1.75.0"

# Install dependencies
yum install -y curl libseccomp-devel openssl openssl-devel skopeo clang clang-devel
Expand All @@ -17,7 +16,3 @@ yum install -y device-mapper-devel-1.02.197-2.el9.ppc64le.rpm
curl https://dl.google.com/go/go${GO_VERSION}.linux-ppc64le.tar.gz -o go${GO_VERSION}.linux-ppc64le.tar.gz && \
rm -rf /usr/local/go && tar -C /usr/local -xzf go${GO_VERSION}.linux-ppc64le.tar.gz && \
rm -f go${GO_VERSION}.linux-ppc64le.tar.gz

# Install Rust
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${RUST_VERSION}
rustup target add powerpc64le-unknown-linux-gnu
5 changes: 2 additions & 3 deletions src/cloud-api-adaptor/podvm-mkosi/Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
include ../Makefile.defaults

ATTESTER ?= none
ARCH ?= $(subst x86_64,amd64,$(shell uname -m))
BUILDER = fedora-binaries-builder-$(ARCH)
SE_BOOT ?= false
Expand Down Expand Up @@ -34,7 +33,6 @@ fedora-binaries-builder:
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg ARCH=$(ARCH) \
--build-arg PROTOC_VERSION=$(PROTOC_VERSION) \
--build-arg RUST_VERSION=$(RUST_VERSION) \
--build-arg YQ_VERSION=$(YQ_VERSION) \
--build-arg YQ_CHECKSUM=$(YQ_CHECKSUM) \
--build-arg YQ_ARCH=$(ARCH) \
Expand All @@ -51,8 +49,9 @@ ifeq ($(IS_DEBIAN),true)
docker buildx use default
endif
docker buildx build \
--progress=plain \
--build-arg BUILDER_IMG=$(BUILDER) \
--build-arg ATTESTER=$(ATTESTER) \
--build-arg TEE_PLATFORM=$(TEE_PLATFORM) \
--build-arg PAUSE_REPO=$(PAUSE_REPO) \
--build-arg PAUSE_VERSION=$(PAUSE_VERSION) \
--build-arg PAUSE_BIN=$(PAUSE_BIN) \
Expand Down
2 changes: 1 addition & 1 deletion src/cloud-api-adaptor/podvm-mkosi/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ Another issue is s390x does not support UEFI. Instead, we can first use **mkosi*
It requires a **s390x host** to build s390x image with make commands:
```
make fedora-binaries-builder
ATTESTER=se-attester make binaries
TEE_PLATFORM=se-attester make binaries
make image
# SE_BOOT=true make image
# make image-debug
Expand Down
4 changes: 2 additions & 2 deletions src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ ARG GUEST_COMPONENTS_VERSION
ARG GUEST_COMPONENTS_REPO
# By default AA will be built with the `all-attesters` feature,
# which doesn't compile ootb on ubuntu.
ARG ATTESTER=none
ARG TEE_PLATFORM=none
# If not provided, uses system architecture
ARG ARCH
#This is the name of the policy file under
Expand All @@ -33,7 +33,7 @@ ENV CLOUD_PROVIDER=${CLOUD_PROVIDER}
ENV PODVM_DISTRO=${PODVM_DISTRO}
ENV GUEST_COMPONENTS_VERSION=${GUEST_COMPONENTS_VERSION}
ENV GUEST_COMPONENTS_REPO=${GUEST_COMPONENTS_REPO}
ENV ATTESTER=${ATTESTER}
ENV TEE_PLATFORM=${TEE_PLATFORM}
ENV ARCH=${ARCH}
ENV DEFAULT_AGENT_POLICY_FILE=${DEFAULT_AGENT_POLICY_FILE}

Expand Down
4 changes: 2 additions & 2 deletions src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.fedora
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ ARG GUEST_COMPONENTS_VERSION
ARG GUEST_COMPONENTS_REPO
# By default AA will be built with the `all-attesters` feature,
# which doesn't compile on fedora.
ARG ATTESTER=none
ARG TEE_PLATFORM=none
# If not provided, uses system architecture
ARG ARCH
#This is the name of the policy file under
Expand All @@ -34,7 +34,7 @@ ENV CLOUD_PROVIDER=${CLOUD_PROVIDER}
ENV PODVM_DISTRO=${PODVM_DISTRO}
ENV GUEST_COMPONENTS_VERSION=${GUEST_COMPONENTS_VERSION}
ENV GUEST_COMPONENTS_REPO=${GUEST_COMPONENTS_REPO}
ENV ATTESTER=${ATTESTER}
ENV TEE_PLATFORM=${TEE_PLATFORM}
ENV ARCH=${ARCH}
ENV DEFAULT_AGENT_POLICY_FILE=${DEFAULT_AGENT_POLICY_FILE}
ENV IMAGE_NAME=${IMAGE_NAME}
Expand Down
4 changes: 2 additions & 2 deletions src/cloud-api-adaptor/podvm/Dockerfile.podvm_binaries.rhel
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ FROM ${BUILDER_IMG} AS podvm_builder
ARG PODVM_DISTRO=rhel
# By default AA will be built with the `all-attesters` feature,
# which doesn't compile ootb on RHEL.
ARG ATTESTER=none
ARG TEE_PLATFORM=none
# If not provided, uses system architecture
ARG ARCH
#This is the name of the policy file under
Expand All @@ -27,7 +27,7 @@ ENV PAUSE_REPO=${PAUSE_REPO}
ENV PAUSE_VERSION=${PAUSE_VERSION}
ENV PAUSE_BIN=${PAUSE_BIN}
ENV PODVM_DISTRO=${PODVM_DISTRO}
ENV ATTESTER=${ATTESTER}
ENV TEE_PLATFORM=${TEE_PLATFORM}
ENV ARCH=${ARCH}
ENV DEFAULT_AGENT_POLICY_FILE=${DEFAULT_AGENT_POLICY_FILE}

Expand Down
8 changes: 1 addition & 7 deletions src/cloud-api-adaptor/podvm/Dockerfile.podvm_builder
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,11 @@ ARG YQ_ARCH="amd64"
ARG PROTOC_ARCH="x86_64"
ARG GO_VERSION
ARG PROTOC_VERSION
ARG RUST_VERSION
ARG YQ_VERSION
ARG YQ_CHECKSUM
ARG ORAS_VERSION
# Without setting ENV gh-action is failing to use the correct values
ENV GO_VERSION=${GO_VERSION}
ENV RUST_VERSION=${RUST_VERSION}
ENV PROTOC_VERSION=${PROTOC_VERSION}
ENV ARCH=${ARCH}
ENV YQ_ARCH=${YQ_ARCH}
Expand Down Expand Up @@ -48,11 +46,7 @@ ADD https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${Y
RUN echo "${YQ_CHECKSUM#sha256:} /usr/local/bin/yq" | sha256sum -c
RUN chmod a+x /usr/local/bin/yq

ENV PATH="/root/.cargo/bin:/usr/local/go/bin:$PATH"

ADD https://sh.rustup.rs rustup
RUN chmod a+x rustup && ./rustup -y --default-toolchain ${RUST_VERSION} \
&& rustup target add x86_64-unknown-linux-musl && ln -sf /usr/bin/g++ /bin/musl-g++
ENV PATH="/usr/local/go/bin:$PATH"

ADD https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip protoc-${PROTOC_VERSION}-linux-x86_64.zip
RUN unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local && rm -f protoc-${PROTOC_VERSION}-linux-x86_64.zip
Expand Down
6 changes: 1 addition & 5 deletions src/cloud-api-adaptor/podvm/Dockerfile.podvm_builder.fedora
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ ARG YQ_ARCH="amd64"
ARG PROTOC_ARCH="x86_64"
ARG GO_VERSION
ARG PROTOC_VERSION
ARG RUST_VERSION
ARG YQ_VERSION
ARG YQ_CHECKSUM
ARG ORAS_VERSION
Expand All @@ -32,10 +31,7 @@ ADD https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${Y
RUN echo "${YQ_CHECKSUM#sha256:} /usr/local/bin/yq" | sha256sum -c
RUN chmod a+x /usr/local/bin/yq

ENV PATH="/root/.cargo/bin:/usr/local/go/bin:$PATH"

ADD https://sh.rustup.rs rustup
RUN chmod a+x rustup && ./rustup -y --default-toolchain ${RUST_VERSION}
ENV PATH="/usr/local/go/bin:$PATH"

ADD https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
RUN unzip protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip -d /usr/local && rm -f protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
Expand Down
6 changes: 1 addition & 5 deletions src/cloud-api-adaptor/podvm/Dockerfile.podvm_builder.rhel
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ ARG YQ_ARCH="amd64"
ARG PROTOC_ARCH="x86_64"
ARG GO_VERSION
ARG PROTOC_VERSION
ARG RUST_VERSION
ARG PACKER_VERSION
ARG ORAS_VERSION
ARG YQ_VERSION
Expand All @@ -24,7 +23,6 @@ ARG ACTIVATION_KEY

# Without setting ENV gh-action is failing to use the correct values
ENV GO_VERSION=${GO_VERSION}
ENV RUST_VERSION=${RUST_VERSION}
ENV PROTOC_VERSION=${PROTOC_VERSION}
ENV PROTOC_ARCH=${PROTOC_ARCH}
ENV ARCH=${ARCH}
Expand Down Expand Up @@ -74,9 +72,7 @@ RUN ln -s /usr/libexec/qemu-kvm /usr/bin/qemu-system-$(uname -m)
RUN git clone https://github.com/canonical/cloud-utils
RUN cd cloud-utils && make install

RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain "${RUST_VERSION}"

ENV PATH="/root/.cargo/bin:/usr/local/go/bin:$PATH"
ENV PATH="/usr/local/go/bin:$PATH"

RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip && \
unzip protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip -d /usr/local && rm -f protoc-${PROTOC_VERSION}-linux-${PROTOC_ARCH}.zip
Expand Down
3 changes: 0 additions & 3 deletions src/cloud-api-adaptor/podvm/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ ifndef IMAGE_CHECKSUM
$(error "IMAGE_CHECKSUM is not defined")
endif

KATA_AGENT_SRC := ../../kata-containers/src/agent
STATIC_LIBSECCOMP_BUILDER := ../../kata-containers/ci/install_libseccomp.sh
AGENT_PROTOCOL_FORWARDER_SRC := ../

QEMU_MACHINE_TYPE_s390x := s390-ccw-virtio
Expand Down Expand Up @@ -105,4 +103,3 @@ clean:
rm -fr "$(SKOPEO_SRC)" "$(UMOCI_SRC)" "$(PAUSE_SRC)" "$(FILES_DIR)/$(PAUSE_BUNDLE)"
rm -fr "$(OPA_SRC)"


Loading

0 comments on commit 4de6783

Please sign in to comment.