docker: add support for crio based e2e tests

A new parameter CONTAINER_RUNTIME is added. Default value is "containerd".
Set the value to "crio" to execute e2e tests on a crio based kind cluster.

Signed-off-by: Pradipta Banerjee <[email protected]>

src/cloud-api-adaptor/docker/README.md

@@ -117,6 +117,9 @@ This will create a two node kind cluster, automatically download the pod VM
 image mentioned in the `provision_docker.properties` file and run the tests. On
 completion of the test, the kind cluster will be automatically deleted.
 
+If you want to run the tests on a crio based kind cluster, then update `CONTAINER_RUNTIME` to `crio`
+in the `provision_docker.properties` file.
+
 > **Note:**  To overcome docker rate limiting issue or to download images from private registries,
 create a `config.json` file under `/tmp` with your registry secrets.
 
@@ -142,7 +145,8 @@ This auth string needs to be used in `/tmp/config.json` as shown below:
 ```
 
 If you want to use a different location for the registry secret, then remember to update the same
-in the `src/cloud-api-adaptor/docker/kind-config.yaml` file.
+in the `src/cloud-api-adaptor/docker/kind-config.yaml` file if using `containerd` or
+in the `src/cloud-api-adaptor/docker/kind-config-crio.yaml` file if using `crio`.
 
 > **Note:** If you have executed the tests with `TEST_TEARDOWN=no`, then you'll
 > need to manually delete the `kind` created cluster by running the following
@@ -152,6 +156,7 @@ in the `src/cloud-api-adaptor/docker/kind-config.yaml` file.
 kind delete cluster --name peer-pods
 ```
 
+
 ## Run sample application
 
 ### Ensure runtimeclass is present

src/cloud-api-adaptor/docker/kind-config-crio.yaml

@@ -0,0 +1,41 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  disableDefaultCNI: true # disable kindnet
+  podSubnet: 192.168.0.0/16 # set to Calico's default subnet
+nodes:
+  - role: control-plane
+    # Instructions to build - https://github.com/cri-o/cri-o/blob/main/tutorials/crio-in-kind.md#build-node-image
+    image: quay.io/confidential-containers/kind-crio:v1.29.4
+    extraMounts:
+      # The config.json file contains the registry secrets that you might
+      # need to pull images from a private registry or docker registry to avoid
+      # rate limiting.
+      - hostPath: /tmp/config.json
+        containerPath: /var/lib/kubelet/config.json
+    kubeadmConfigPatches:
+      - |
+        kind: InitConfiguration
+        nodeRegistration:
+          criSocket: unix:///var/run/crio/crio.sock
+      - |
+        kind: JoinConfiguration
+        nodeRegistration:
+          criSocket: unix:///var/run/crio/crio.sock
+  - role: worker
+    image: quay.io/confidential-containers/kind-crio:v1.29.4
+    extraMounts:
+      - hostPath: /var/run/docker.sock
+        containerPath: /var/run/docker.sock
+      - hostPath: /var/lib/docker
+        containerPath: /var/lib/docker
+      # The config.json file contains the registry secrets that you might
+      # need to pull images from a private registry or docker registry to avoid
+      # rate limiting.
+      - hostPath: /tmp/config.json
+        containerPath: /var/lib/kubelet/config.json
+    kubeadmConfigPatches:
+      - |
+        kind: JoinConfiguration
+        nodeRegistration:
+          criSocket: unix:///var/run/crio/crio.sock

src/cloud-api-adaptor/docker/kind_cluster.sh

@@ -8,6 +8,7 @@ newgrp docker <<EOF
 # delete: deletes a kind cluster
 
 CLUSTER_NAME="${CLUSTER_NAME:-peer-pods}"
+KIND_CONFIG_FILE="kind-config.yaml"
 
 if [ "$1" == "create" ]; then
     # Check if kind is installed
@@ -25,9 +26,17 @@ if [ "$1" == "create" ]; then
     sudo sysctl fs.inotify.max_user_watches=524288
     sudo sysctl fs.inotify.max_user_instances=512
 
+    # Create a kind cluster
+    echo "runtime: " "\$CONTAINER_RUNTIME"
+
+    if [ "$CONTAINER_RUNTIME" == "crio" ]; then
+        echo "Creating a kind cluster with crio runtime"
+        KIND_CONFIG_FILE="kind-config-crio.yaml"
+    fi
+
     # Create a kind cluster
     echo "Creating a kind cluster"
-    kind create cluster --name "\$CLUSTER_NAME" --config kind-config.yaml || exit 1
+    kind create cluster --name "\$CLUSTER_NAME" --config "\$KIND_CONFIG_FILE" || exit 1
 
     # Deploy calico
     kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml || exit 1

src/cloud-api-adaptor/test/provisioner/docker/provision_common.go

@@ -31,31 +31,33 @@ type DockerInstallOverlay struct {
 }
 
 type DockerProperties struct {
-	DockerHost  string
-	ApiVer      string
-	ClusterName string
-	NetworkName string
-	PodvmImage  string
-	CaaImage    string
-	CaaImageTag string
-	KbsImage    string
-	KbsImageTag string
+	DockerHost       string
+	ApiVer           string
+	ClusterName      string
+	NetworkName      string
+	PodvmImage       string
+	CaaImage         string
+	CaaImageTag      string
+	KbsImage         string
+	KbsImageTag      string
+	ContainerRuntime string
 }
 
 var DockerProps = &DockerProperties{}
 
 func initDockerProperties(properties map[string]string) error {
 
 	DockerProps = &DockerProperties{
-		DockerHost:  properties["DOCKER_HOST"],
-		ApiVer:      properties["DOCKER_API_VERSION"],
-		ClusterName: properties["CLUSTER_NAME"],
-		NetworkName: properties["DOCKER_NETWORK_NAME"],
-		PodvmImage:  properties["DOCKER_PODVM_IMAGE"],
-		CaaImage:    properties["CAA_IMAGE"],
-		CaaImageTag: properties["CAA_IMAGE_TAG"],
-		KbsImage:    properties["KBS_IMAGE"],
-		KbsImageTag: properties["KBS_IMAGE_TAG"],
+		DockerHost:       properties["DOCKER_HOST"],
+		ApiVer:           properties["DOCKER_API_VERSION"],
+		ClusterName:      properties["CLUSTER_NAME"],
+		NetworkName:      properties["DOCKER_NETWORK_NAME"],
+		PodvmImage:       properties["DOCKER_PODVM_IMAGE"],
+		CaaImage:         properties["CAA_IMAGE"],
+		CaaImageTag:      properties["CAA_IMAGE_TAG"],
+		KbsImage:         properties["KBS_IMAGE"],
+		KbsImageTag:      properties["KBS_IMAGE_TAG"],
+		ContainerRuntime: properties["CONTAINER_RUNTIME"],
 	}
 	return nil
 }
@@ -138,6 +140,7 @@ func (l *DockerProvisioner) GetProperties(ctx context.Context, cfg *envconf.Conf
 		"CAA_IMAGE_TAG":       DockerProps.CaaImageTag,
 		"KBS_IMAGE":           DockerProps.KbsImage,
 		"KBS_IMAGE_TAG":       DockerProps.KbsImageTag,
+		"CONTAINER_RUNTIME":   DockerProps.ContainerRuntime,
 	}
 }
 
@@ -164,8 +167,8 @@ func createKindCluster(workingDir string) error {
 	// TODO: better handle stderr. Messages getting out of order.
 	cmd.Stderr = os.Stderr
 	cmd.Env = os.Environ()
-	// Set CLUSTER_NAME if available. Also unset KUBECONFIG so that the default path is used.
-	cmd.Env = append(cmd.Env, "CLUSTER_NAME="+DockerProps.ClusterName, "KUBECONFIG=")
+	// Set CLUSTER_NAME and CONTAINER_RUNTIME if available. Also unset KUBECONFIG so that the default path is used.
+	cmd.Env = append(cmd.Env, "CLUSTER_NAME="+DockerProps.ClusterName, "KUBECONFIG=", "CONTAINER_RUNTIME="+DockerProps.ContainerRuntime)
 	err := cmd.Run()
 	if err != nil {
 		log.Errorf("Error creating Kind cluster: %v", err)

src/cloud-api-adaptor/test/provisioner/docker/provision_docker.properties

@@ -9,3 +9,6 @@ CAA_IMAGE_TAG=""
 # KBS configs
 KBS_IMAGE=""
 KBS_IMAGE_TAG=""
+
+# either "containerd" or "crio"
+CONTAINER_RUNTIME="containerd"