Skip to content

Commit

Permalink
SecureComms: E2e test SecureComms without KBS
Browse files Browse the repository at this point in the history
Add support for e2e testing SecureComms without KBS

Signed-off-by: David Hadas <[email protected]>
  • Loading branch information
davidhadas authored and davidhIBM committed Dec 12, 2024
1 parent b9679f0 commit 2331371
Show file tree
Hide file tree
Showing 7 changed files with 129 additions and 41 deletions.
7 changes: 7 additions & 0 deletions src/cloud-api-adaptor/libvirt/config_libvirt.sh
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,13 @@ echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties
# switch to the appropriate e2e test and add configs to libvirt.properties as needed
case $TEST_E2E_SECURE_COMMS in

"withoutKbs")
echo "processing withoutKbs"
echo "SECURE_COMMS=\"true\"" >> libvirt.properties
echo "SECURE_COMMS_NO_TRUSTEE=\"true\"" >> libvirt.properties
echo "INITDATA=\"\"" >> libvirt.properties
;;

*)
echo "processing none"
echo "SECURE_COMMS=\"false\"" >> libvirt.properties
Expand Down
2 changes: 1 addition & 1 deletion src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"container_runtime": ["containerd", "crio"],
"secure_comms": ["none"],
"secure_comms": ["none", "withoutKbs"],
"os": ["ubuntu"],
"provider": ["generic"],
"arch": ["amd64"]
Expand Down
15 changes: 15 additions & 0 deletions src/cloud-api-adaptor/test/e2e/assessment_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,21 @@ func VerifyAlternateImage(ctx context.Context, t *testing.T, client klient.Clien
return nil
}

func VerifySecureCommsActivated(ctx context.Context, t *testing.T, client klient.Client, pod *v1.Pod) error {
nodeName, err := GetNodeNameFromPod(ctx, client, pod)
if err != nil {
return fmt.Errorf("VerifySecureCommsConnected: GetNodeNameFromPod failed with %v", err)
}

expectedSuccessMessage := "Using PP SecureComms"
err = VerifyCaaPodLogContains(ctx, t, client, nodeName, expectedSuccessMessage)
if err != nil {
return fmt.Errorf("VerifySecureCommsConnected: failed: %v", err)
}
t.Logf("PodVM was brought up using SecureComms")
return nil
}

func VerifyCaaPodLogContains(ctx context.Context, t *testing.T, client klient.Client, nodeName, expected string) error {
caaPod, err := getCaaPod(ctx, client, t, nodeName)
if err != nil {
Expand Down
9 changes: 9 additions & 0 deletions src/cloud-api-adaptor/test/e2e/assessment_runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ type ExtraPod struct {
testCommands []TestCommand
}

var testCase_secureComms_isActive bool

type TestCase struct {
testing *testing.T
testEnv env.Environment
Expand Down Expand Up @@ -420,6 +422,13 @@ func (tc *TestCase) Run() {
t.Errorf("VerifyAlternateImage failed: %v", err)
}
}

if testCase_secureComms_isActive {
err := VerifySecureCommsActivated(ctx, t, client, tc.pod)
if err != nil {
t.Errorf("VerifySecureCommsActivated failed: %v", err)
}
}
}

if tc.extraPods != nil {
Expand Down
8 changes: 8 additions & 0 deletions src/cloud-api-adaptor/test/e2e/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,14 @@ func TestMain(m *testing.M) {

// Get properties
props := provisioner.GetProperties(ctx, cfg)
log.Infof("Do setup test only props: %v", props)
log.Infof("Do setup test only prop SECURE_COMMS value: %s", props["SECURE_COMMS"])
if props["SECURE_COMMS"] == "true" {
testCase_secureComms_isActive = true
log.Info("Do setup secureComms is active")
}
testCase_secureComms_isActive = true
log.Info("Do setup test only secureComms")

// Set CONTAINER_RUNTIME env variable if present in the properties
// Default value is containerd.
Expand Down
115 changes: 77 additions & 38 deletions src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,21 @@ const AlternateVolumeName = "another-podvm-base.qcow2"

// LibvirtProvisioner implements the CloudProvisioner interface for Libvirt.
type LibvirtProvisioner struct {
conn *libvirt.Connect // Libvirt connection
containerRuntime string // Name of the container runtime
network string // Network name
ssh_key_file string // SSH key file used to connect to Libvirt
storage string // Storage pool name
uri string // Libvirt URI
wd string // libvirt's directory path on this repository
volumeName string // Podvm volume name
clusterName string // Cluster name
tunnelType string // Tunnel Type
vxlanPort string // VXLAN port number
conn *libvirt.Connect // Libvirt connection
containerRuntime string // Name of the container runtime
network string // Network name
ssh_key_file string // SSH key file used to connect to Libvirt
storage string // Storage pool name
uri string // Libvirt URI
wd string // libvirt's directory path on this repository
volumeName string // Podvm volume name
clusterName string // Cluster name
tunnelType string // Tunnel Type
vxlanPort string // VXLAN port number
secure_comms string // Activate CAA SECURE_COMMS
secure_comms_no_trustee string // Deactivate Trustee mode in SECURE_COMMS
secure_comms_kbs_addr string // KBS URL
initdata string // InitData
}

// LibvirtInstallOverlay implements the InstallOverlay interface
Expand Down Expand Up @@ -95,19 +99,47 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e
vxlanPort = properties["vxlan_port"]
}

secure_comms := "false"
if properties["SECURE_COMMS"] != "" {
secure_comms = properties["SECURE_COMMS"]
}

log.Tracef("NewLibvirtProvisioner properties %v", properties)

log.Tracef("NewLibvirtProvisioner secure_comms %s", secure_comms)

secure_comms_kbs_addr := ""
if properties["SECURE_COMMS_KBS_ADDR"] != "" {
secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"]
}

secure_comms_no_trustee := "false"
if properties["SECURE_COMMS_NO_TRUSTEE"] != "" {
secure_comms_no_trustee = properties["SECURE_COMMS_NO_TRUSTEE"]
}

initdata := ""
if properties["INITDATA"] != "" {
initdata = properties["INITDATA"]
}

// TODO: Check network and storage are not nil?
return &LibvirtProvisioner{
conn: conn,
containerRuntime: properties["container_runtime"],
network: network,
ssh_key_file: ssh_key_file,
storage: storage,
uri: uri,
wd: wd,
volumeName: vol_name,
clusterName: clusterName,
tunnelType: tunnelType,
vxlanPort: vxlanPort,
conn: conn,
containerRuntime: properties["container_runtime"],
network: network,
ssh_key_file: ssh_key_file,
storage: storage,
uri: uri,
wd: wd,
volumeName: vol_name,
clusterName: clusterName,
tunnelType: tunnelType,
vxlanPort: vxlanPort,
secure_comms: secure_comms,
secure_comms_kbs_addr: secure_comms_kbs_addr,
secure_comms_no_trustee: secure_comms_no_trustee,
initdata: initdata,
}, nil
}

Expand Down Expand Up @@ -212,14 +244,18 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config)

func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string {
return map[string]string{
"CONTAINER_RUNTIME": l.containerRuntime,
"network": l.network,
"podvm_volume": l.volumeName,
"ssh_key_file": l.ssh_key_file,
"storage": l.storage,
"uri": l.uri,
"tunnel_type": l.tunnelType,
"vxlan_port": l.vxlanPort,
"CONTAINER_RUNTIME": l.containerRuntime,
"network": l.network,
"podvm_volume": l.volumeName,
"ssh_key_file": l.ssh_key_file,
"storage": l.storage,
"uri": l.uri,
"tunnel_type": l.tunnelType,
"vxlan_port": l.vxlanPort,
"SECURE_COMMS": l.secure_comms,
"SECURE_COMMS_KBS_ADDR": l.secure_comms_kbs_addr,
"SECURE_COMMS_NO_TRUSTEE": l.secure_comms_no_trustee,
"INITDATA": l.initdata,
}
}

Expand Down Expand Up @@ -326,14 +362,17 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,

// Mapping the internal properties to ConfigMapGenerator properties and their default values.
mapProps := map[string][2]string{
"network": {"default", "LIBVIRT_NET"},
"storage": {"default", "LIBVIRT_POOL"},
"pause_image": {"", "PAUSE_IMAGE"},
"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
"uri": {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
"tunnel_type": {"", "TUNNEL_TYPE"},
"vxlan_port": {"", "VXLAN_PORT"},
"INITDATA": {"", "INITDATA"},
"network": {"default", "LIBVIRT_NET"},
"storage": {"default", "LIBVIRT_POOL"},
"pause_image": {"", "PAUSE_IMAGE"},
"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
"uri": {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
"tunnel_type": {"", "TUNNEL_TYPE"},
"vxlan_port": {"", "VXLAN_PORT"},
"INITDATA": {"", "INITDATA"},
"SECURE_COMMS": {"", "SECURE_COMMS"},
"SECURE_COMMS_NO_TRUSTEE": {"", "SECURE_COMMS_NO_TRUSTEE"},
"SECURE_COMMS_KBS_ADDR": {"", "SECURE_COMMS_KBS_ADDR"},
}

for k, v := range mapProps {
Expand Down
14 changes: 12 additions & 2 deletions src/cloud-api-adaptor/test/provisioner/provision.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/BurntSushi/toml"
Expand Down Expand Up @@ -220,7 +221,6 @@ func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error
wait.WithTimeout(time.Minute*1)); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -295,7 +295,17 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props
}
}

fmt.Printf("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName())
log.Trace("CAA ConfigMap:\n")
caaConfigMap := exec.Command("kubectl", "get", "cm", "peer-pods-cm", "-n", "confidential-containers-system", "-o", "yaml")
caaConfigMap.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
caaConfigMapOut := new(strings.Builder)
caaConfigMap.Stdout = caaConfigMapOut
if err = caaConfigMap.Run(); err != nil {
return err
}
log.Tracef("%v, CAA ConfigMap: \n%s", caaConfigMap, caaConfigMapOut.String())

log.Infof("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName())
if err = wait.For(conditions.New(resources).ResourcesFound(&nodev1.RuntimeClassList{Items: []nodev1.RuntimeClass{*p.runtimeClass}}),
wait.WithTimeout(time.Second*60)); err != nil {
return err
Expand Down

0 comments on commit 2331371

Please sign in to comment.