From be301d2fb5e5353fa7da290051ca5a2521f8e227 Mon Sep 17 00:00:00 2001 From: David Hadas Date: Sat, 21 Sep 2024 13:42:41 -0500 Subject: [PATCH] SecureComms: E2e test SecureComms without KBS Add support for e2e testing SecureComms without KBS Signed-off-by: David Hadas --- .../libvirt/config_libvirt.sh | 14 ++- .../libvirt/e2e_matrix_libvirt.json | 2 +- .../test/e2e/assessment_helpers.go | 15 +++ .../test/e2e/assessment_runner.go | 15 +++ .../test/e2e/common_suite.go | 12 ++ .../test/e2e/libvirt_test.go | 5 + src/cloud-api-adaptor/test/e2e/main_test.go | 4 + .../provisioner/libvirt/provision_common.go | 115 ++++++++++++------ .../test/provisioner/provision.go | 14 ++- 9 files changed, 153 insertions(+), 43 deletions(-) diff --git a/src/cloud-api-adaptor/libvirt/config_libvirt.sh b/src/cloud-api-adaptor/libvirt/config_libvirt.sh index 5ad41be22..2649b0ce5 100755 --- a/src/cloud-api-adaptor/libvirt/config_libvirt.sh +++ b/src/cloud-api-adaptor/libvirt/config_libvirt.sh @@ -92,7 +92,7 @@ installK8sclis() { fi } -TEST_E2E_SECURE_COMMS=${TEST_E2E_SECURE_COMMS:-none}. +TEST_E2E_SECURE_COMMS=${TEST_E2E_SECURE_COMMS:-none} echo "SECURE_COMMS is ${TEST_E2E_SECURE_COMMS}" echo "Installing Go..." @@ -123,10 +123,20 @@ echo "libvirt_ssh_key_file=\"id_rsa\"" >> libvirt.properties echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties # switch to the appropriate e2e test and add configs to libvirt.properties as needed -case $TEST_E2E_SECURE_COMMS in +case ${TEST_E2E_SECURE_COMMS} in + + withoutKbs) + echo "processing withoutKbs" + echo "SECURE_COMMS=\"true\"" >> libvirt.properties + echo "SECURE_COMMS_NO_TRUSTEE=\"true\"" >> libvirt.properties + echo "INITDATA=\"\"" >> libvirt.properties + ;; *) echo "processing none" echo "SECURE_COMMS=\"false\"" >> libvirt.properties ;; esac + +echo "Content of libvirt.properties" +cat libvirt.properties diff --git a/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json b/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json index d0ed40b54..58238feff 100644 --- a/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json +++ b/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json @@ -1,6 +1,6 @@ { "container_runtime": ["containerd", "crio"], - "secure_comms": ["none"], + "secure_comms": ["none", "withoutKbs"], "os": ["ubuntu"], "provider": ["generic"], "arch": ["amd64"] diff --git a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go index 10976f332..2ea652c55 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go @@ -292,6 +292,21 @@ func VerifyAlternateImage(ctx context.Context, t *testing.T, client klient.Clien return nil } +func VerifySecureCommsActivated(ctx context.Context, t *testing.T, client klient.Client, pod *v1.Pod) error { + nodeName, err := GetNodeNameFromPod(ctx, client, pod) + if err != nil { + return fmt.Errorf("VerifySecureCommsConnected: GetNodeNameFromPod failed with %v", err) + } + + expectedSuccessMessage := "Using PP SecureComms" + err = VerifyCaaPodLogContains(ctx, t, client, nodeName, expectedSuccessMessage) + if err != nil { + return fmt.Errorf("VerifySecureCommsConnected: failed: %v", err) + } + t.Logf("PodVM was brought up using SecureComms") + return nil +} + func VerifyCaaPodLogContains(ctx context.Context, t *testing.T, client klient.Client, nodeName, expected string) error { caaPod, err := getCaaPod(ctx, client, t, nodeName) if err != nil { diff --git a/src/cloud-api-adaptor/test/e2e/assessment_runner.go b/src/cloud-api-adaptor/test/e2e/assessment_runner.go index 15e75f723..1af006cb6 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_runner.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_runner.go @@ -48,6 +48,8 @@ type ExtraPod struct { testCommands []TestCommand } +var testCase_secureComms_isActive bool + type TestCase struct { testing *testing.T testEnv env.Environment @@ -72,6 +74,7 @@ type TestCase struct { expectedInstanceType string isNydusSnapshotter bool alternateImageName string + secureCommsIsActive bool } func (tc *TestCase) WithConfigMap(configMap *v1.ConfigMap) *TestCase { @@ -134,6 +137,11 @@ func (tc *TestCase) WithAlternateImage(alternateImageName string) *TestCase { return tc } +func (tc *TestCase) WithSecureCommsIsActive() *TestCase { + tc.secureCommsIsActive = true + return tc +} + func (pod *ExtraPod) WithTestCommands(TestCommands []TestCommand) *ExtraPod { pod.testCommands = TestCommands return pod @@ -420,6 +428,13 @@ func (tc *TestCase) Run() { t.Errorf("VerifyAlternateImage failed: %v", err) } } + + if tc.secureCommsIsActive { + err := VerifySecureCommsActivated(ctx, t, client, tc.pod) + if err != nil { + t.Errorf("VerifySecureCommsActivated failed: %v", err) + } + } } if tc.extraPods != nil { diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 9bce01655..c6f430578 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -35,6 +35,18 @@ func DoTestCreateSimplePod(t *testing.T, e env.Environment, assert CloudAssert) } } +func DoTestLibvirtCreateSimplePodWithSecureCommsIsValid(t *testing.T, e env.Environment, assert CloudAssert) { + if !testCase_secureComms_isActive { + t.Skip("Skip - SecureComms is configured to be inactive - no need to test") + } + pod := NewBusyboxPodWithName(E2eNamespace, "simple-test-with-security-comms-is-active").GetPodOrFatal(t) + if isTestOnCrio() { + NewTestCase(t, e, "SimplePeerPodWithSecureComms", assert, "PodVM is created with secure comms").WithPod(pod).WithSecureCommsIsActive().Run() + } else { + NewTestCase(t, e, "SimplePeerPodWithSecureComms", assert, "PodVM is created with secure comms").WithPod(pod).WithNydusSnapshotter().WithSecureCommsIsActive().Run() + } +} + func DoTestDeleteSimplePod(t *testing.T, e env.Environment, assert CloudAssert) { pod := NewBusyboxPodWithName(E2eNamespace, "deletion-test").GetPodOrFatal(t) duration := assert.DefaultTimeout() diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 4a3b33b8f..46e726e5e 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -18,6 +18,11 @@ func TestLibvirtCreateSimplePod(t *testing.T) { DoTestCreateSimplePod(t, testEnv, assert) } +func TestLibvirtCreateSimplePodWithSecureCommsIsValid(t *testing.T) { + assert := LibvirtAssert{} + DoTestLibvirtCreateSimplePodWithSecureCommsIsValid(t, testEnv, assert) +} + func TestLibvirtCreatePodWithConfigMap(t *testing.T) { assert := LibvirtAssert{} DoTestCreatePodWithConfigMap(t, testEnv, assert) diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index 330b5d591..133fb42e2 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -123,6 +123,10 @@ func TestMain(m *testing.M) { // Get properties props := provisioner.GetProperties(ctx, cfg) + if props["SECURE_COMMS"] == "true" { + testCase_secureComms_isActive = true + log.Info("Do setup secureComms is active") + } // Set CONTAINER_RUNTIME env variable if present in the properties // Default value is containerd. diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index 6cc9dd1c2..8f9100ac1 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -23,17 +23,21 @@ const AlternateVolumeName = "another-podvm-base.qcow2" // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt. type LibvirtProvisioner struct { - conn *libvirt.Connect // Libvirt connection - containerRuntime string // Name of the container runtime - network string // Network name - ssh_key_file string // SSH key file used to connect to Libvirt - storage string // Storage pool name - uri string // Libvirt URI - wd string // libvirt's directory path on this repository - volumeName string // Podvm volume name - clusterName string // Cluster name - tunnelType string // Tunnel Type - vxlanPort string // VXLAN port number + conn *libvirt.Connect // Libvirt connection + containerRuntime string // Name of the container runtime + network string // Network name + ssh_key_file string // SSH key file used to connect to Libvirt + storage string // Storage pool name + uri string // Libvirt URI + wd string // libvirt's directory path on this repository + volumeName string // Podvm volume name + clusterName string // Cluster name + tunnelType string // Tunnel Type + vxlanPort string // VXLAN port number + secure_comms string // Activate CAA SECURE_COMMS + secure_comms_no_trustee string // Deactivate Trustee mode in SECURE_COMMS + secure_comms_kbs_addr string // KBS URL + initdata string // InitData } // LibvirtInstallOverlay implements the InstallOverlay interface @@ -95,19 +99,47 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e vxlanPort = properties["vxlan_port"] } + secure_comms := "false" + if properties["SECURE_COMMS"] != "" { + secure_comms = properties["SECURE_COMMS"] + } + + log.Tracef("NewLibvirtProvisioner properties %v", properties) + + log.Tracef("NewLibvirtProvisioner secure_comms %s", secure_comms) + + secure_comms_kbs_addr := "" + if properties["SECURE_COMMS_KBS_ADDR"] != "" { + secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"] + } + + secure_comms_no_trustee := "false" + if properties["SECURE_COMMS_NO_TRUSTEE"] != "" { + secure_comms_no_trustee = properties["SECURE_COMMS_NO_TRUSTEE"] + } + + initdata := "" + if properties["INITDATA"] != "" { + initdata = properties["INITDATA"] + } + // TODO: Check network and storage are not nil? return &LibvirtProvisioner{ - conn: conn, - containerRuntime: properties["container_runtime"], - network: network, - ssh_key_file: ssh_key_file, - storage: storage, - uri: uri, - wd: wd, - volumeName: vol_name, - clusterName: clusterName, - tunnelType: tunnelType, - vxlanPort: vxlanPort, + conn: conn, + containerRuntime: properties["container_runtime"], + network: network, + ssh_key_file: ssh_key_file, + storage: storage, + uri: uri, + wd: wd, + volumeName: vol_name, + clusterName: clusterName, + tunnelType: tunnelType, + vxlanPort: vxlanPort, + secure_comms: secure_comms, + secure_comms_kbs_addr: secure_comms_kbs_addr, + secure_comms_no_trustee: secure_comms_no_trustee, + initdata: initdata, }, nil } @@ -212,14 +244,18 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string { return map[string]string{ - "CONTAINER_RUNTIME": l.containerRuntime, - "network": l.network, - "podvm_volume": l.volumeName, - "ssh_key_file": l.ssh_key_file, - "storage": l.storage, - "uri": l.uri, - "tunnel_type": l.tunnelType, - "vxlan_port": l.vxlanPort, + "CONTAINER_RUNTIME": l.containerRuntime, + "network": l.network, + "podvm_volume": l.volumeName, + "ssh_key_file": l.ssh_key_file, + "storage": l.storage, + "uri": l.uri, + "tunnel_type": l.tunnelType, + "vxlan_port": l.vxlanPort, + "SECURE_COMMS": l.secure_comms, + "SECURE_COMMS_KBS_ADDR": l.secure_comms_kbs_addr, + "SECURE_COMMS_NO_TRUSTEE": l.secure_comms_no_trustee, + "INITDATA": l.initdata, } } @@ -326,14 +362,17 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "tunnel_type": {"", "TUNNEL_TYPE"}, - "vxlan_port": {"", "VXLAN_PORT"}, - "INITDATA": {"", "INITDATA"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "tunnel_type": {"", "TUNNEL_TYPE"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "INITDATA": {"", "INITDATA"}, + "SECURE_COMMS": {"", "SECURE_COMMS"}, + "SECURE_COMMS_NO_TRUSTEE": {"", "SECURE_COMMS_NO_TRUSTEE"}, + "SECURE_COMMS_KBS_ADDR": {"", "SECURE_COMMS_KBS_ADDR"}, } for k, v := range mapProps { diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index 3c69b8080..f5551e464 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/BurntSushi/toml" @@ -220,7 +221,6 @@ func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error wait.WithTimeout(time.Minute*1)); err != nil { return err } - return nil } @@ -295,7 +295,17 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props } } - fmt.Printf("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName()) + log.Trace("CAA ConfigMap:\n") + caaConfigMap := exec.Command("kubectl", "get", "cm", "peer-pods-cm", "-n", "confidential-containers-system", "-o", "yaml") + caaConfigMap.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + caaConfigMapOut := new(strings.Builder) + caaConfigMap.Stdout = caaConfigMapOut + if err = caaConfigMap.Run(); err != nil { + return err + } + log.Tracef("%v, CAA ConfigMap: \n%s", caaConfigMap, caaConfigMapOut.String()) + + log.Infof("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName()) if err = wait.For(conditions.New(resources).ResourcesFound(&nodev1.RuntimeClassList{Items: []nodev1.RuntimeClass{*p.runtimeClass}}), wait.WithTimeout(time.Second*60)); err != nil { return err