SecureComms: E2e test SecureComms without KBS

Add support for e2e testing SecureComms without KBS

Signed-off-by: David Hadas <[email protected]>

src/cloud-api-adaptor/libvirt/README.md

@@ -217,6 +217,7 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2
 * ``TEST_E2E_TIMEOUT`` - test timeout
 * ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow
 * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh))
+* ``TEST_CAA_LOG`` - whether to log the CAA at the end of the tests run
 
 # Delete Confidential Containers and cloud-api-adaptor from the cluster
 

src/cloud-api-adaptor/libvirt/config_libvirt.sh

@@ -125,6 +125,13 @@ echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties
 # switch to the appropriate e2e test and add configs to libvirt.properties as needed
 case $TEST_E2E_SECURE_COMMS in
 
+  "withoutKbs")
+    echo "processing withoutKbs"
+    echo "SECURE_COMMS=\"true\"" >> libvirt.properties
+    echo "SECURE_COMMS_NO_TRUSTEE=\"true\"" >> libvirt.properties
+    echo "INITDATA=\"\"" >> libvirt.properties
+    ;;
+
   *)
     echo "processing none"
     echo "SECURE_COMMS=\"false\"" >> libvirt.properties

src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json

@@ -1,5 +1,5 @@
 {
-    "secure_comms": ["none"],
+    "secure_comms": ["none", "withoutKbs"],
     "os": ["ubuntu"],
     "provider": ["generic"],
     "arch": ["amd64"]

src/cloud-api-adaptor/test/e2e/main_test.go

@@ -5,7 +5,9 @@ package e2e
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"os/exec"
 	"testing"
 
 	pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner"
@@ -58,6 +60,12 @@ func TestMain(m *testing.M) {
 	// unless it is running with an in-cluster configuration.
 	testEnv = env.New()
 
+	// TEST_CAA_LOG is an option variable which specifies whether the CAA Log should be
+	// presented at the end of the test or not
+	shouldCaaLog := true
+	if os.Getenv("TEST_CAA_LOG") == "no" {
+		shouldCaaLog = false
+	}
 	// TEST_TEARDOWN is an option variable which specifies whether the teardown code path
 	// should run or not.
 	shouldTeardown := true
@@ -192,6 +200,18 @@ func TestMain(m *testing.M) {
 
 	// Run *once* after the tests.
 	testEnv.Finish(func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+
+		if shouldCaaLog {
+			log.Info("CAA LOG: STARTING\n")
+			caaLogTailCmd := exec.Command("kubectl", "logs", "daemonset/cloud-api-adaptor-daemonset", "-n", "confidential-containers-system")
+			caaLogTailCmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
+			caaLogTailCmd.Stdout = os.Stdout
+			caaLogTailCmd.Stderr = os.Stderr
+			if err := caaLogTailCmd.Run(); err != nil {
+				return ctx, err
+			}
+		}
+
 		if !shouldTeardown {
 			return ctx, nil
 		}

src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go

@@ -23,14 +23,18 @@ const AlternateVolumeName = "another-podvm-base.qcow2"
 
 // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt.
 type LibvirtProvisioner struct {
-	conn         *libvirt.Connect // Libvirt connection
-	network      string           // Network name
-	ssh_key_file string           // SSH key file used to connect to Libvirt
-	storage      string           // Storage pool name
-	uri          string           // Libvirt URI
-	wd           string           // libvirt's directory path on this repository
-	volumeName   string           // Podvm volume name
-	clusterName  string           // Cluster name
+	conn                    *libvirt.Connect // Libvirt connection
+	network                 string           // Network name
+	ssh_key_file            string           // SSH key file used to connect to Libvirt
+	storage                 string           // Storage pool name
+	uri                     string           // Libvirt URI
+	wd                      string           // libvirt's directory path on this repository
+	volumeName              string           // Podvm volume name
+	clusterName             string           // Cluster name
+	secure_comms            string           // Activate CAA SECURE_COMMS
+	secure_comms_no_trustee string           // Deactivate Trustee mode in SECURE_COMMS
+	secure_comms_kbs_addr   string           // KBS URL
+	initdata                string           // InitData
 }
 
 // LibvirtInstallOverlay implements the InstallOverlay interface
@@ -82,16 +86,40 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e
 		clusterName = properties["cluster_name"]
 	}
 
+	secure_comms := "false"
+	if properties["SECURE_COMMS"] != "" {
+		secure_comms = properties["SECURE_COMMS"]
+	}
+
+	secure_comms_kbs_addr := ""
+	if properties["SECURE_COMMS_KBS_ADDR"] != "" {
+		secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"]
+	}
+
+	secure_comms_no_trustee := "false"
+	if properties["SECURE_COMMS_NO_TRUSTEE"] != "" {
+		secure_comms_no_trustee = properties["SECURE_COMMS_NO_TRUSTEE"]
+	}
+
+	initdata := ""
+	if properties["INITDATA"] != "" {
+		initdata = properties["INITDATA"]
+	}
+
 	// TODO: Check network and storage are not nil?
 	return &LibvirtProvisioner{
-		conn:         conn,
-		network:      network,
-		ssh_key_file: ssh_key_file,
-		storage:      storage,
-		uri:          uri,
-		wd:           wd,
-		volumeName:   vol_name,
-		clusterName:  clusterName,
+		conn:                    conn,
+		network:                 network,
+		ssh_key_file:            ssh_key_file,
+		storage:                 storage,
+		uri:                     uri,
+		wd:                      wd,
+		volumeName:              vol_name,
+		clusterName:             clusterName,
+		secure_comms:            secure_comms,
+		secure_comms_kbs_addr:   secure_comms_kbs_addr,
+		secure_comms_no_trustee: secure_comms_no_trustee,
+		initdata:                initdata,
 	}, nil
 }
 
@@ -196,11 +224,15 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config)
 
 func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string {
 	return map[string]string{
-		"network":      l.network,
-		"podvm_volume": l.volumeName,
-		"ssh_key_file": l.ssh_key_file,
-		"storage":      l.storage,
-		"uri":          l.uri,
+		"network":                 l.network,
+		"podvm_volume":            l.volumeName,
+		"ssh_key_file":            l.ssh_key_file,
+		"storage":                 l.storage,
+		"uri":                     l.uri,
+		"SECURE_COMMS":            l.secure_comms,
+		"SECURE_COMMS_KBS_ADDR":   l.secure_comms_kbs_addr,
+		"SECURE_COMMS_NO_TRUSTEE": l.secure_comms_no_trustee,
+		"INITDATA":                l.initdata,
 	}
 }
 
@@ -307,13 +339,16 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 
 	// Mapping the internal properties to ConfigMapGenerator properties and their default values.
 	mapProps := map[string][2]string{
-		"network":      {"default", "LIBVIRT_NET"},
-		"storage":      {"default", "LIBVIRT_POOL"},
-		"pause_image":  {"", "PAUSE_IMAGE"},
-		"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
-		"uri":          {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
-		"vxlan_port":   {"", "VXLAN_PORT"},
-		"INITDATA":     {"", "INITDATA"},
+		"network":                 {"default", "LIBVIRT_NET"},
+		"storage":                 {"default", "LIBVIRT_POOL"},
+		"pause_image":             {"", "PAUSE_IMAGE"},
+		"podvm_volume":            {"", "LIBVIRT_VOL_NAME"},
+		"uri":                     {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
+		"vxlan_port":              {"", "VXLAN_PORT"},
+		"INITDATA":                {"", "INITDATA"},
+		"SECURE_COMMS":            {"", "SECURE_COMMS"},
+		"SECURE_COMMS_NO_TRUSTEE": {"", "SECURE_COMMS_NO_TRUSTEE"},
+		"SECURE_COMMS_KBS_ADDR":   {"", "SECURE_COMMS_KBS_ADDR"},
 	}
 
 	for k, v := range mapProps {

src/cloud-api-adaptor/test/provisioner/provision.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/BurntSushi/toml"
@@ -220,7 +221,6 @@ func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error
 		wait.WithTimeout(time.Minute*1)); err != nil {
 		return err
 	}
-
 	return nil
 }
 
@@ -295,7 +295,17 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props
 		}
 	}
 
-	fmt.Printf("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName())
+	log.Trace("CAA ConfigMap:\n")
+	caaConfigMap := exec.Command("kubectl", "get", "cm", "peer-pods-cm", "-n", "confidential-containers-system", "-o", "yaml")
+	caaConfigMap.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
+	caaConfigMapOut := new(strings.Builder)
+	caaConfigMap.Stdout = caaConfigMapOut
+	if err = caaConfigMap.Run(); err != nil {
+		return err
+	}
+	log.Tracef("%v, CAA ConfigMap: \n%s", caaConfigMap, caaConfigMapOut.String())
+
+	log.Infof("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName())
 	if err = wait.For(conditions.New(resources).ResourcesFound(&nodev1.RuntimeClassList{Items: []nodev1.RuntimeClass{*p.runtimeClass}}),
 		wait.WithTimeout(time.Second*60)); err != nil {
 		return err