diff --git a/src/cloud-api-adaptor/docker/README.md b/src/cloud-api-adaptor/docker/README.md index f383523d0..40e2b0e10 100644 --- a/src/cloud-api-adaptor/docker/README.md +++ b/src/cloud-api-adaptor/docker/README.md @@ -13,8 +13,8 @@ The `docker` provider simulates a pod VM inside a docker container. Docker engine version 26+ supports API 1.44. Ensure you complete the [post install steps](https://docs.docker.com/engine/install/linux-postinstall/) if using non-root user - - + + - Kubernetes cluster ## Build CAA pod-VM image @@ -26,13 +26,13 @@ export CLOUD_PROVIDER=docker ``` - Build the required pod VM binaries - + ```bash cd src/cloud-api-adaptor/docker/image make ``` -This will build the required binaries inside a container and place +This will build the required binaries inside a container and place it under `resources/binaries-tree` - Build the pod VM image @@ -44,8 +44,9 @@ cd ../../ This will build the podvm docker image. By default the image is named `quay.io/confidential-containers/podvm-docker-image`. -For quick changes you can just build the binaries of podvm components and update `./resources/binaries-tree/usr/local/bin` with the new -components and run `make image` to build a new podvm image. +For quick changes you can just build the binaries of podvm components and +update `./resources/binaries-tree/usr/local/bin` with the new components and +run `make image` to build a new podvm image. You can download a ready-to-use image on your worker node. @@ -56,7 +57,6 @@ docker pull quay.io/confidential-containers/podvm-docker-image Note that before you can spin up a pod, the podvm image must be available on the K8s worker node with the docker engine installed. - ## Build CAA container image > **Note**: If you have made changes to the CAA code and you want to deploy those changes then follow [these instructions](https://github.com/confidential-containers/cloud-api-adaptor/blob/main/src/cloud-api-adaptor/install/README.md#building-custom-cloud-api-adaptor-image) to build the container image from the root of this repository. @@ -65,7 +65,6 @@ with the docker engine installed. The following [`kustomization.yaml`](../install/overlays/docker/kustomization.yaml) is used. - ### Deploy CAA on the Kubernetes cluster Run the following command to deploy CAA: @@ -80,7 +79,82 @@ For changing the CAA image to your custom built image (eg. `quay.io/myuser/cloud you can use the following: ```bash -kubectl set image ds/cloud-api-adaptor-daemonset -n confidential-containers-system cloud-api-adaptor-con=quay.io/myuser/cloud-api-adaptor +export CAA_IMAGE=quay.io/myuser/cloud-api-adaptor +kubectl set image ds/cloud-api-adaptor-daemonset -n confidential-containers-system cloud-api-adaptor-con="$CAA_IMAGE" +``` + +## Running the CAA e2e tests + +### Test Prerequisites + +To run the tests, use a test system with at least 8GB RAM and 4vCPUs. +Ubuntu 22.04 has been tested. Other Linux distros should work, but it has not +been tested. + +Following software prerequisites are needed on the test system: + +- make +- go +- yq +- kubectl +- kind +- docker + +A `prereqs.sh` helper script is available under `src/cloud-api-adaptor/docker` to install/uninstall the prerequisites. + + +> **Note:** If using the `prereqs.sh` helper script to install the +> prerequisites, then reload the shell to ensure new permissions +are in place to run `docker` and other commands. + +### Test Execution + +In order to run the tests, edit the file `src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties` +and update the `CAA_IMAGE` and `CAA_IMAGE_TAG` variables with your custom CAA image and tag. + +You can run the CAA e2e [tests/e2e/README.md](../test/e2e/README.md) by running the following command: + +```sh +make TEST_PODVM_IMAGE= TEST_PROVISION=yes CLOUD_PROVIDER=docker TEST_PROVISION_FILE=$(pwd)/test/provisioner/docker/provision_docker.properties test-e2e +``` + +This will create a two node kind cluster, automatically download the pod VM +image mentioned in the `provision_docker.properties` file and run the tests. On +completion of the test, the kind cluster will be automatically deleted. + +> **Note:** To overcome docker rate limiting issue or to download images from private registries, +create a `config.json` file under `/tmp` with your registry secrets. + +For example: +If your docker registry user is `someuser` and password is `somepassword` then create the auth string +as shown below: + +```sh +echo -n "someuser:somepassword" | base64 +c29tZXVzZXI6c29tZXBhc3N3b3Jk +``` + +This auth string needs to be used in `/tmp/config.json` as shown below: + +```sh +{ + "auths": { + "https://index.docker.io/v1/": { + "auth": "c29tZXVzZXI6c29tZXBhc3N3b3Jk" + } + } +} +``` + +If you want to use a different location for the registry secret, then remember to update the same +in the `src/cloud-api-adaptor/docker/kind-config.yaml` file. + +> **Note:** If you have executed the tests with `TEST_TEARDOWN=no`, then you'll +> need to manually delete the `kind` created cluster by running the following +> command: + +```sh +kind delete cluster --name peer-pods ``` ## Run sample application @@ -151,7 +225,7 @@ nginx-dbc79c87-jt49h 1/1 Running 1 (3m22s ago) 3m29s $ docker ps CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES e60b768b847d quay.io/confidential-containers/podvm-docker-image "/usr/local/bin/entr…" 3 minutes ago Up 3 minutes 15150/tcp podvm-nginx-dbc79c87-jt49h-b9361aef -``` +``` For debugging you can use docker commands like `docker ps`, `docker logs`, `docker exec`. @@ -161,3 +235,21 @@ For debugging you can use docker commands like `docker ps`, `docker logs`, `dock kubectl delete deployment nginx ``` +## Troubleshooting + +When using `containerd` and `nydus-snapshotter` you might encounter pod creation failure due to +issues with unpacking of image. Check the `nydus-snapshotter` troubleshooting [doc](../docs/troubleshooting/nydus-snapshotter.md). + +In order to login to the worker node you can use either of the following approaches + +```sh +kubectl debug node/peer-pods-worker -it --image=busybox + +# chroot /host +``` + +or + +```sh +docker exec -it peer-pods-worker bash +``` \ No newline at end of file diff --git a/src/cloud-api-adaptor/docker/kind-config.yaml b/src/cloud-api-adaptor/docker/kind-config.yaml new file mode 100644 index 000000000..3b12ea511 --- /dev/null +++ b/src/cloud-api-adaptor/docker/kind-config.yaml @@ -0,0 +1,27 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: true # disable kindnet + podSubnet: 192.168.0.0/16 # set to Calico's default subnet +nodes: + - role: control-plane + # Same image version as used for pod VM base image + image: kindest/node:v1.29.4 + extraMounts: + # The config.json file contains the registry secrets that you might + # need to pull images from a private registry or docker registry to avoid + # rate limiting. + - hostPath: /tmp/config.json + containerPath: /var/lib/kubelet/config.json + - role: worker + image: kindest/node:v1.29.4 + extraMounts: + - hostPath: /var/run/docker.sock + containerPath: /var/run/docker.sock + - hostPath: /var/lib/docker + containerPath: /var/lib/docker + # The config.json file contains the registry secrets that you might + # need to pull images from a private registry or docker registry to avoid + # rate limiting. + - hostPath: /tmp/config.json + containerPath: /var/lib/kubelet/config.json diff --git a/src/cloud-api-adaptor/docker/kind_cluster.sh b/src/cloud-api-adaptor/docker/kind_cluster.sh new file mode 100755 index 000000000..696750d60 --- /dev/null +++ b/src/cloud-api-adaptor/docker/kind_cluster.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# Ref: https://stackoverflow.com/questions/299728/how-do-you-use-newgrp-in-a-script-then-stay-in-that-group-when-the-script-exits +newgrp docker <&2 + exit 1 +} + +# function to check if passwordless sudo is enabled +check_sudo() { + if sudo -n true 2>/dev/null; then + echo "Passwordless sudo is enabled." + else + error_exit "Passwordless sudo is not enabled. Please enable passwordless sudo." + fi +} + +# function to compare versions +# returns true if $1 < $2 +version_lt() { + [ "$1" != "$2" ] && [ "$(printf '%s\n' "$@" | sort -V | head -n 1)" == "$1" ] +} + +# function to install OS packages +# the packages are available in the variable REQUIRED_OS_PACKAGES +# the function will install the packages using the package manager +# Following are the packages that are installed: +# make +install_os_packages() { + # Define the required OS packages + REQUIRED_OS_PACKAGES=( + "make" + ) + + # Install the required OS packages + for package in "${REQUIRED_OS_PACKAGES[@]}"; do + if [[ -x "$(command -v "${package}")" ]]; then + echo "Package ${package} is already installed. Skipping." + continue + else + echo "Installing ${package}..." + # Update a hidden file to record what was installed. This is useful for uninstallation + echo "${package}" >>"${os_packages_file}" + if [[ -x "$(command -v apt-get)" ]]; then + # shellcheck disable=SC2015 + sudo apt-get update && + sudo apt-get install -y "${package}" || + error_exit "Failed to install ${package}" + elif [[ -x "$(command -v dnf)" ]]; then + sudo dnf install -y "${package}" || + error_exit "Failed to install ${package}" + else + error_exit "Unsupported OS" + fi + fi + done + + echo "All OS packages installed successfully." + +} + +# function to download and install binary packages. +# the packages, their respective download locations and compression +# are available in the variable REQUIRED_BINARY_PACKAGES +# the function will download the packages, extract them and install them in /usr/local/bin +# Following are the packages that are installed: +# yq=https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64 +# kubectl=https://storage.googleapis.com/kubernetes-release/release/v1.27.11/bin/linux/amd64/kubectl +# kind=https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64 + +install_binary_packages() { + # Define the required binary packages + REQUIRED_BINARY_PACKAGES=( + "yq=https://github.com/mikefarah/yq/releases/download/v4.44.2/yq_linux_amd64" + "kubectl=https://storage.googleapis.com/kubernetes-release/release/v1.27.11/bin/linux/amd64/kubectl" + "kind=https://kind.sigs.k8s.io/dl/v0.23.0/kind-linux-amd64" + ) + + # Specify the installation directory + local install_dir="/usr/local/bin" + + # Install the required binary packages + for package_info in "${REQUIRED_BINARY_PACKAGES[@]}"; do + IFS='=' read -r package_name package_url <<<"${package_info}" + download_path="/tmp/${package_name}" + + if [[ -x "${install_dir}/${package_name}" ]]; then + echo "Package ${package_name} is already installed. Skipping." + continue + else + echo "Downloading ${package_name}..." + # Update a hidden file to record what was installed. This is useful for uninstallation + echo "${package_name}" >>"${bin_packages_file}" + curl -sSL "${package_url}" -o "${download_path}" || + error_exit "Failed to download ${package_name}" + + echo "Extracting ${package_name}..." + if [[ "${package_url}" == *.tar.gz ]]; then + sudo tar -xf "${download_path}" -C "${install_dir}" || + error_exit "Failed to extract ${package_name}" + # If not a tar.gz file, then it is a binary file + else + echo "${package_name} is binary file." + sudo mv "${download_path}" "${install_dir}/${package_name}" || + error_exit "Failed to move ${package_name} to ${install_dir}" + fi + + echo "Marking ${install_dir}/${package_name} executable" + sudo chmod +x "${install_dir}/${package_name}" || + error_exit "Failed to mark ${package_name} executable" + + echo "Cleaning up..." + rm -f "${download_path}" + fi + done + + echo "All binary packages installed successfully." + +} + +# function to install golang +install_golang() { + echo "Path to versions.yaml: $VERSIONS_YAML_PATH" + + # When installing, use the required min go version from the versions.yaml file + REQUIRED_GO_VERSION="$(yq '.tools.golang' "$VERSIONS_YAML_PATH")" + # Check if Go is already installed + if [[ -x "$(command -v go)" ]]; then + echo "Go is already installed" + # Check if the installed version is less than the required version + installed_go_version=$(v=$(go version | awk '{print $3}') && echo "${v#go}") + + if version_lt "$installed_go_version" "$REQUIRED_GO_VERSION"; then + echo "Warning: Found ${installed_go_version}, is lower than our required $REQUIRED_GO_VERSION" + echo "Please remove the existing go version and run this script again." + exit 1 + else + echo "Found ${installed_go_version}, good to go" + fi + else + # Install Go + echo "Installing Go" + curl -fsSL https://go.dev/dl/go${REQUIRED_GO_VERSION}.linux-amd64.tar.gz -o go.tar.gz || error_exit "Failed to download Go" + sudo tar -C /usr/local -xzf go.tar.gz || error_exit "Failed to extract Go" + touch "$go_file" + rm -f go.tar.gz + fi + +} + +# function to uninstall golang +uninstall_golang() { + # Check if Go is installed + if [[ ! -x "$(command -v go)" ]]; then + echo "Go is not installed" + return + fi + + # Uninstall Go + echo "Uninstalling Go" + + # Uninstall only if go_file exists + if [[ ! -f "$go_file" ]]; then + echo "Go was not installed using this script. Skipping uninstallation." + return + fi + + sudo rm -rf /usr/local/go + + # Remove the file that records the installed packages + rm -f "$go_file" +} + +# function to install docker +install_docker() { + # Check if Docker is already installed + if [[ -x "$(command -v docker)" ]]; then + echo "Docker is already installed" + else + # Install Docker + echo "Installing Docker" + curl -fsSL https://get.docker.com -o get-docker.sh || error_exit "Failed to download Docker installation script" + sudo sh get-docker.sh || error_exit "Failed to install Docker" + touch "$docker_file" + rm -f get-docker.sh + sudo groupadd docker + sudo usermod -aG docker "$USER" + fi +} + +#function to uninstall the binary packages +uninstall_binary_packages() { + # Remove the installed binary packages + # The packages are available under bin_packages_file file + # The function will remove the packages from install_dir + + # Check if the file exists + if [[ ! -f "${bin_packages_file}" ]]; then + echo "No binary packages to uninstall." + return + fi + + # Read the file and uninstall the packages + while IFS= read -r package_name; do + if [[ -x "${install_dir}/${package_name}" ]]; then + echo "Uninstalling ${package_name}..." + sudo rm -f "${install_dir}/${package_name}" || + error_exit "Failed to uninstall ${package_name}" + + else + echo "Package ${package_name} is not installed. Skipping." + fi + done <"${bin_packages_file}" + + # Remove the file that records the installed packages + rm -f "${bin_packages_file}" + + echo "All binary packages uninstalled successfully." +} + +# function to uninstall the OS packages +uninstall_os_packages() { + # Remove the installed Os packages + # The packages are available under os_packages_file file + # The function will remove the packages using the package manager + + # Check if the file exists + if [[ ! -f "${os_packages_file}" ]]; then + echo "No OS packages to uninstall." + return + fi + + # Read the file and uninstall the packages + while IFS= read -r package_name; do + if [[ -x "$(command -v "${package_name}")" ]]; then + echo "Uninstalling ${package_name}..." + if [[ -x "$(command -v apt-get)" ]]; then + sudo apt-get purge -y "${package_name}" || + error_exit "Failed to uninstall ${package_name}" + elif [[ -x "$(command -v dnf)" ]]; then + sudo dnf remove -y "${package_name}" || + error_exit "Failed to uninstall ${package_name}" + else + error_exit "Unsupported OS" + fi + else + echo "Package ${package_name} is not installed. Skipping." + fi + done <"${os_packages_file}" + + # Remove the file that records the installed packages + rm -f "${os_packages_file}" + + echo "All OS packages uninstalled successfully." +} + +# function to uninstall docker +uninstall_docker() { + # Check if Docker is installed + if [[ ! -x "$(command -v docker)" ]]; then + echo "Docker is not installed" + return + fi + + # Uninstall Docker + echo "Uninstalling Docker" + + # Uninstall only if docker_file exists + if [[ ! -f "$docker_file" ]]; then + echo "Docker was not installed using this script. Skipping uninstallation." + return + fi + # Check if OS is Ubuntu + if [[ -x "$(command -v apt-get)" ]]; then + sudo apt-get purge -y docker-ce docker-ce-cli containerd.io \ + docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras || error_exit "Failed to uninstall Docker" + sudo rm -rf /var/lib/docker + sudo rm -rf /var/lib/containerd + elif [[ -x "$(command -v dnf)" ]]; then + sudo dnf remove -y docker-ce docker-ce-cli containerd.io \ + docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras || error_exit "Failed to uninstall Docker" + sudo rm -rf /var/lib/docker + sudo rm -rf /var/lib/containerd + fi + + # Remove the file that records the installed packages + rm -f "$docker_file" +} + +#function to set PATH to include /usr/local/bin and /usr/local/go/bin +set_path() { + + # If bin_packages_file exists, then add /usr/local/bin to PATH + if [[ -f "${bin_packages_file}" ]]; then + # Add /usr/local/bin to PATH + if [[ ":$PATH:" == *":/usr/local/bin:"* ]]; then + echo "/usr/local/bin is already in PATH. Skipping." + else + echo "Adding /usr/local/bin to PATH..." + # shellcheck disable=SC2016 + echo 'export PATH=$PATH:/usr/local/bin' >>"$HOME"/.bashrc + fi + fi + + # If go_file exists, then add /usr/local/go/bin to PATH + if [[ -f "${go_file}" ]]; then + # Add /usr/local/go/bin to PATH + if [[ ":$PATH:" == *":/usr/local/go/bin:"* ]]; then + echo "/usr/local/go/bin is already in PATH. Skipping." + else + echo "Adding /usr/local/go/bin to PATH..." + # shellcheck disable=SC2016 + echo 'export PATH=$PATH:/usr/local/go/bin' >>"$HOME"/.bashrc + fi + fi + + # Reload the bashrc file + # shellcheck source=/dev/null + source "$HOME"/.bashrc + +} + +if [[ "$(uname)" != "Linux" || "$(uname -m)" != "x86_64" ]]; then + error_exit "This script is only for Linux x86_64 OS." +fi + +# Main function +if [[ "$1" == "install" ]]; then + check_sudo + install_os_packages + install_binary_packages + install_golang + install_docker + set_path +elif [[ "$1" == "uninstall" ]]; then + check_sudo + uninstall_binary_packages + uninstall_os_packages + uninstall_golang + uninstall_docker +else + error_exit "Invalid argument. Please provide either 'install' or 'uninstall'." +fi diff --git a/src/cloud-api-adaptor/docs/troubleshooting/nydus-snapshotter.md b/src/cloud-api-adaptor/docs/troubleshooting/nydus-snapshotter.md new file mode 100644 index 000000000..b0fc938d2 --- /dev/null +++ b/src/cloud-api-adaptor/docs/troubleshooting/nydus-snapshotter.md @@ -0,0 +1,53 @@ +# Some tips for debugging nydus-snapshotter related issues + +## Check nydus configuration + +When using `containerd` runtime, CoCo operator configures `nydus-snapshotter` to handle in-guest +container image pull. + +Login to the worker node and run the following command to verify if `nydus-snapshotter` is set up successfully. + +```sh +ctr -a /run/containerd/containerd.sock plugin ls | grep nydus +``` + +You should see an o/p like the following: + +```sh +io.containerd.snapshotter.v1 nydus - ok +``` + +## Pod creation fails with "error unpacking image" + +Sometimes when creating a pod you might encounter the following error: + +```sh +Warning Failed 8m51s (x7 over 10m) kubelet Error: failed to create containerd container: error unpacking image: failed to extract layer sha256:d51af96cf93e225825efd484ea457f867cb2b967f7415b9a3b7e65a2f803838a: failed to get reader from content store: content digest sha256:ec562eabd705d25bfea8c8d79e4610775e375524af00552fe871d3338261563c: not found +``` + +To resolve this, login to the worker node and explicitly fetch the image using the following command: + +```sh +export image=<_set_> +ctr -n k8s.io content fetch $image +``` + +Here is a concrete example describing how to download the images in case of the above error +when running e2e tests: + +```sh +images=( + "quay.io/prometheus/busybox:latest" + "quay.io/confidential-containers/test-images:testworkdir" + "docker.io/library/nginx:latest" + "docker.io/curlimages/curl:8.4.0" + "quay.io/curl/curl:latest" +) + +# Loop through each image in the list +# and download the image +for image in "${images[@]}"; do + ctr -n k8s.io content fetch $image +done +``` + diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod index c1b250523..75c220792 100644 --- a/src/cloud-api-adaptor/go.mod +++ b/src/cloud-api-adaptor/go.mod @@ -50,6 +50,7 @@ require ( github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers v0.9.0-alpha1 github.com/confidential-containers/cloud-api-adaptor/src/peerpod-ctrl v0.9.0-alpha1 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f + github.com/docker/docker v25.0.5+incompatible github.com/golang-jwt/jwt/v5 v5.2.1 github.com/moby/sys/mountinfo v0.7.1 github.com/pelletier/go-toml/v2 v2.1.0 @@ -102,7 +103,6 @@ require ( github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v25.0.5+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/src/cloud-api-adaptor/test/e2e/docker_common.go b/src/cloud-api-adaptor/test/e2e/docker_common.go new file mode 100644 index 000000000..217f8fbdc --- /dev/null +++ b/src/cloud-api-adaptor/test/e2e/docker_common.go @@ -0,0 +1,53 @@ +//go:build docker + +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +// DockerAssert implements the CloudAssert interface for Docker. +type DockerAssert struct { + // TODO: create the connection once on the initializer. + //conn client.Connect +} + +func (c DockerAssert) DefaultTimeout() time.Duration { + return 1 * time.Minute +} + +func (l DockerAssert) HasPodVM(t *testing.T, id string) { + conn, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + t.Fatal(err) + } + + // Check if the container is running + containers, err := conn.ContainerList(context.Background(), types.ContainerListOptions{}) + if err != nil { + t.Fatal(err) + } + + for _, container := range containers { + if strings.Contains(container.Names[0], id) { + return + } + } + + // It didn't find the PodVM if it reached here. + t.Error("PodVM was not created") +} + +func (l DockerAssert) GetInstanceType(t *testing.T, podName string) (string, error) { + // Get Instance Type of PodVM + return "", nil +} diff --git a/src/cloud-api-adaptor/test/e2e/docker_test.go b/src/cloud-api-adaptor/test/e2e/docker_test.go new file mode 100644 index 000000000..13423d9e8 --- /dev/null +++ b/src/cloud-api-adaptor/test/e2e/docker_test.go @@ -0,0 +1,110 @@ +//go:build docker + +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "testing" + + _ "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner/docker" +) + +func TestDockerCreateSimplePod(t *testing.T) { + assert := DockerAssert{} + DoTestCreateSimplePod(t, testEnv, assert) +} + +func TestDockerCreatePodWithConfigMap(t *testing.T) { + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestCreatePodWithConfigMap(t, testEnv, assert) +} + +func TestDockerCreatePodWithSecret(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePodWithSecret(t, testEnv, assert) +} + +func TestDockerCreatePeerPodContainerWithExternalIPAccess(t *testing.T) { + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestCreatePeerPodContainerWithExternalIPAccess(t, testEnv, assert) + +} + +func TestDockerCreatePeerPodWithJob(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodWithJob(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckUserLogs(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckUserLogs(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckWorkDirLogs(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckWorkDirLogs(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithImageOnly(t *testing.T) { + // This test is causing issues on CI with instability, so skip until we can resolve this. + // See https://github.com/confidential-containers/cloud-api-adaptor/issues/1831 + SkipTestOnCI(t) + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckEnvVariableLogsWithImageOnly(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithDeploymentOnly(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckEnvVariableLogsWithDeploymentOnly(t, testEnv, assert) +} + +func TestDockerCreatePeerPodAndCheckEnvVariableLogsWithImageAndDeployment(t *testing.T) { + // This test is causing issues on CI with instability, so skip until we can resolve this. + // See https://github.com/confidential-containers/cloud-api-adaptor/issues/1831 + assert := DockerAssert{} + DoTestCreatePeerPodAndCheckEnvVariableLogsWithImageAndDeployment(t, testEnv, assert) +} + +func TestDockerCreateNginxDeployment(t *testing.T) { + assert := DockerAssert{} + DoTestNginxDeployment(t, testEnv, assert) +} + +/* +Failing due to issues will pulling image (ErrImagePull) +func TestDockerCreatePeerPodWithLargeImage(t *testing.T) { + assert := DockerAssert{} + DoTestCreatePeerPodWithLargeImage(t, testEnv, assert) +} +*/ + +func TestDockerDeletePod(t *testing.T) { + assert := DockerAssert{} + DoTestDeleteSimplePod(t, testEnv, assert) +} + +func TestDockerPodToServiceCommunication(t *testing.T) { + assert := DockerAssert{} + DoTestPodToServiceCommunication(t, testEnv, assert) +} + +func TestDockerPodsMTLSCommunication(t *testing.T) { + assert := DockerAssert{} + DoTestPodsMTLSCommunication(t, testEnv, assert) +} + +func TestDockerKbsKeyRelease(t *testing.T) { + if !isTestWithKbs() { + t.Skip("Skipping kbs related test as kbs is not deployed") + } + _ = keyBrokerService.EnableKbsCustomizedPolicy("deny_all.rego") + assert := DockerAssert{} + t.Parallel() + DoTestKbsKeyReleaseForFailure(t, testEnv, assert) + _ = keyBrokerService.EnableKbsCustomizedPolicy("allow_all.rego") + DoTestKbsKeyRelease(t, testEnv, assert) +} diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision.go b/src/cloud-api-adaptor/test/provisioner/docker/provision.go new file mode 100644 index 000000000..325a6940c --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision.go @@ -0,0 +1,15 @@ +//go:build docker + +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner" +) + +func init() { + pv.NewProvisionerFunctions["docker"] = NewDockerProvisioner + pv.NewInstallOverlayFunctions["docker"] = NewDockerInstallOverlay +} diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go new file mode 100644 index 000000000..676646b31 --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_common.go @@ -0,0 +1,263 @@ +// (C) Copyright Confidential Containers Contributors +// SPDX-License-Identifier: Apache-2.0 + +package docker + +import ( + "context" + "fmt" + + "os" + "os/exec" + "path/filepath" + + pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner" + "github.com/containerd/containerd/reference" + "github.com/docker/docker/client" + log "github.com/sirupsen/logrus" + + "sigs.k8s.io/e2e-framework/pkg/envconf" +) + +// DockerProvisioner implements the CloudProvisioner interface for Docker. +type DockerProvisioner struct { + conn *client.Client + wd string // docker's directory path on this repository +} + +// DockerInstallOverlay implements the InstallOverlay interface +type DockerInstallOverlay struct { + Overlay *pv.KustomizeOverlay +} + +type DockerProperties struct { + DockerHost string + ApiVer string + ClusterName string + NetworkName string + PodvmImage string + CaaImage string + CaaImageTag string + KbsImage string + KbsImageTag string +} + +var DockerProps = &DockerProperties{} + +func initDockerProperties(properties map[string]string) error { + + DockerProps = &DockerProperties{ + DockerHost: properties["DOCKER_HOST"], + ApiVer: properties["DOCKER_API_VERSION"], + ClusterName: properties["CLUSTER_NAME"], + NetworkName: properties["DOCKER_NETWORK_NAME"], + PodvmImage: properties["DOCKER_PODVM_IMAGE"], + CaaImage: properties["CAA_IMAGE"], + CaaImageTag: properties["CAA_IMAGE_TAG"], + KbsImage: properties["KBS_IMAGE"], + KbsImageTag: properties["KBS_IMAGE_TAG"], + } + return nil +} + +func NewDockerProvisioner(properties map[string]string) (pv.CloudProvisioner, error) { + wd, err := filepath.Abs(filepath.Join("..", "..", "docker")) + if err != nil { + log.Errorf("Error getting the absolute path of the docker directory: %v", err) + return nil, err + } + + if err := initDockerProperties(properties); err != nil { + return nil, err + } + + // set environment variables + os.Setenv("DOCKER_HOST", DockerProps.DockerHost) + + conn, err := client.NewClientWithOpts(client.FromEnv) + if err != nil { + log.Errorf("Error creating the Docker client: %v", err) + return nil, err + } + + return &DockerProvisioner{ + conn: conn, + wd: wd, + }, nil +} + +func (l *DockerProvisioner) CreateCluster(ctx context.Context, cfg *envconf.Config) error { + + // Create kind cluster + if err := createKindCluster(l.wd); err != nil { + log.Errorf("Error creating Kind cluster: %v", err) + return err + } + + home, _ := os.UserHomeDir() + kubeconfig := filepath.Join(home, ".kube/config") + cfg.WithKubeconfigFile(kubeconfig) + + if err := pv.AddNodeRoleWorkerLabel(ctx, DockerProps.ClusterName, cfg); err != nil { + + return fmt.Errorf("labeling nodes: %w", err) + } + + return nil +} + +func (l *DockerProvisioner) DeleteCluster(ctx context.Context, cfg *envconf.Config) error { + + if err := deleteKindCluster(l.wd); err != nil { + log.Errorf("Error deleting Kind cluster: %v", err) + return err + } + return nil +} + +func (l *DockerProvisioner) CreateVPC(ctx context.Context, cfg *envconf.Config) error { + // TODO: delete the resources created on CreateVPC() that currently only checks + // the Docker's storage and network exist. + return nil +} + +func (l *DockerProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) error { + // TODO: delete the resources created on CreateVPC() that currently only checks + // the Docker's storage and network exist. + return nil +} + +func (l *DockerProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string { + return map[string]string{ + "DOCKER_HOST": DockerProps.DockerHost, + "DOCKER_API_VERSION": DockerProps.ApiVer, + "CLUSTER_NAME": DockerProps.ClusterName, + "DOCKER_NETWORK_NAME": DockerProps.NetworkName, + "DOCKER_PODVM_IMAGE": DockerProps.PodvmImage, + "CAA_IMAGE": DockerProps.CaaImage, + "CAA_IMAGE_TAG": DockerProps.CaaImageTag, + "KBS_IMAGE": DockerProps.KbsImage, + "KBS_IMAGE_TAG": DockerProps.KbsImageTag, + } +} + +func (l *DockerProvisioner) UploadPodvm(imagePath string, ctx context.Context, cfg *envconf.Config) error { + + // Download the podvm image from the registry by using docker pull + cmd := exec.Command("/bin/bash", "-c", "docker pull "+DockerProps.PodvmImage) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + log.Errorf("Error pulling the podvm image: %v", err) + return err + } + + return nil +} + +func createKindCluster(workingDir string) error { + // Create kind cluster by executing the script on the node + cmd := exec.Command("/bin/bash", "-c", "./kind_cluster.sh create") + cmd.Dir = workingDir + cmd.Stdout = os.Stdout + // TODO: better handle stderr. Messages getting out of order. + cmd.Stderr = os.Stderr + cmd.Env = os.Environ() + // Set CLUSTER_NAME if available. Also unset KUBECONFIG so that the default path is used. + cmd.Env = append(cmd.Env, "CLUSTER_NAME="+DockerProps.ClusterName, "KUBECONFIG=") + err := cmd.Run() + if err != nil { + log.Errorf("Error creating Kind cluster: %v", err) + return err + } + return nil +} + +func deleteKindCluster(workingDir string) error { + // Delete kind cluster by executing the script on the node + cmd := exec.Command("/bin/bash", "-c", "./kind_cluster.sh delete") + cmd.Dir = workingDir + cmd.Stdout = os.Stdout + // TODO: better handle stderr. Messages getting out of order. + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + log.Errorf("Error deleting Kind cluster: %v", err) + return err + } + return nil +} + +func NewDockerInstallOverlay(installDir, provider string) (pv.InstallOverlay, error) { + overlay, err := pv.NewKustomizeOverlay(filepath.Join(installDir, "overlays", provider)) + if err != nil { + log.Errorf("Error creating the docker provider install overlay: %v", err) + return nil, err + } + + return &DockerInstallOverlay{ + Overlay: overlay, + }, nil +} + +func isDockerKustomizeConfigMapKey(key string) bool { + switch key { + case "CLOUD_PROVIDER", "DOCKER_HOST", "DOCKER_API_VERSION", "DOCKER_PODVM_IMAGE", "DOCKER_NETWORK_NAME", "AA_KBC_PARAMS": + return true + default: + return false + } +} + +func (lio *DockerInstallOverlay) Apply(ctx context.Context, cfg *envconf.Config) error { + return lio.Overlay.Apply(ctx, cfg) +} + +func (lio *DockerInstallOverlay) Delete(ctx context.Context, cfg *envconf.Config) error { + return lio.Overlay.Delete(ctx, cfg) +} + +// Update install/overlays/docker/kustomization.yaml +func (lio *DockerInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, properties map[string]string) error { + + // If a custom image is defined then update it in the kustomization file. + if DockerProps.CaaImage != "" { + spec, err := reference.Parse(DockerProps.CaaImage) + if err != nil { + return fmt.Errorf("parsing image: %w", err) + } + + log.Infof("Updating CAA image with %q", spec.Locator) + if err = lio.Overlay.SetKustomizeImage("cloud-api-adaptor", "newName", spec.Locator); err != nil { + return err + } + } + + if DockerProps.CaaImageTag != "" { + spec, err := reference.Parse(DockerProps.CaaImageTag) + if err != nil { + return fmt.Errorf("parsing image tag: %w", err) + } + log.Infof("Updating CAA image tag with %q", spec.Locator) + if err = lio.Overlay.SetKustomizeImage("cloud-api-adaptor", "newTag", spec.Locator); err != nil { + return err + } + } + + for k, v := range properties { + // configMapGenerator + if isDockerKustomizeConfigMapKey(k) { + if err := lio.Overlay.SetKustomizeConfigMapGeneratorLiteral("peer-pods-cm", k, v); err != nil { + return err + } + } + } + + if err := lio.Overlay.YamlReload(); err != nil { + return err + } + + return nil + +} diff --git a/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties b/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties new file mode 100644 index 000000000..01483011f --- /dev/null +++ b/src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties @@ -0,0 +1,11 @@ +# Docker configs +CLUSTER_NAME="peer-pods" +DOCKER_HOST="unix:///var/run/docker.sock" +DOCKER_PODVM_IMAGE="quay.io/confidential-containers/podvm-docker-image" +DOCKER_NETWORK_NAME="kind" +CAA_IMAGE="" +CAA_IMAGE_TAG="" + +# KBS configs +KBS_IMAGE="" +KBS_IMAGE_TAG=""