SecureComms: E2e test SecureComms without KBS

Add support for e2e testing SecureComms without KBS Signed-off-by: David Hadas <[email protected]>
confidential-containers · Dec 14, 2024 · 81af629 · 81af629
1 parent b9679f0
commit 81af629
Show file tree

Hide file tree

Showing 9 changed files with 141 additions and 43 deletions.
diff --git a/src/cloud-api-adaptor/libvirt/config_libvirt.sh b/src/cloud-api-adaptor/libvirt/config_libvirt.sh
@@ -92,7 +92,7 @@ installK8sclis() {
     fi
 }
 
-TEST_E2E_SECURE_COMMS=${TEST_E2E_SECURE_COMMS:-none}.
+TEST_E2E_SECURE_COMMS=${TEST_E2E_SECURE_COMMS:-none}
 echo "SECURE_COMMS is ${TEST_E2E_SECURE_COMMS}"
 
 echo "Installing Go..."
@@ -123,7 +123,14 @@ echo "libvirt_ssh_key_file=\"id_rsa\"" >> libvirt.properties
 echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties
 
 # switch to the appropriate e2e test and add configs to libvirt.properties as needed
-case $TEST_E2E_SECURE_COMMS in
+case ${TEST_E2E_SECURE_COMMS} in
+
+  withoutKbs)
+    echo "processing withoutKbs"
+    echo "SECURE_COMMS=\"true\"" >> libvirt.properties
+    echo "SECURE_COMMS_NO_TRUSTEE=\"true\"" >> libvirt.properties
+    echo "INITDATA=\"\"" >> libvirt.properties
+    ;;
 
   *)
     echo "processing none"

diff --git a/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json b/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json
@@ -1,6 +1,6 @@
 {
     "container_runtime": ["containerd", "crio"],
-    "secure_comms": ["none"],
+    "secure_comms": ["none", "withoutKbs"],
     "os": ["ubuntu"],
     "provider": ["generic"],
     "arch": ["amd64"]

diff --git a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go
@@ -292,6 +292,21 @@ func VerifyAlternateImage(ctx context.Context, t *testing.T, client klient.Clien
 	return nil
 }
 
+func VerifySecureCommsActivated(ctx context.Context, t *testing.T, client klient.Client, pod *v1.Pod) error {
+	nodeName, err := GetNodeNameFromPod(ctx, client, pod)
+	if err != nil {
+		return fmt.Errorf("VerifySecureCommsConnected: GetNodeNameFromPod failed with %v", err)
+	}
+
+	expectedSuccessMessage := "Using PP SecureComms"
+	err = VerifyCaaPodLogContains(ctx, t, client, nodeName, expectedSuccessMessage)
+	if err != nil {
+		return fmt.Errorf("VerifySecureCommsConnected: failed: %v", err)
+	}
+	t.Logf("PodVM was brought up using SecureComms")
+	return nil
+}
+
 func VerifyCaaPodLogContains(ctx context.Context, t *testing.T, client klient.Client, nodeName, expected string) error {
 	caaPod, err := getCaaPod(ctx, client, t, nodeName)
 	if err != nil {

diff --git a/src/cloud-api-adaptor/test/e2e/assessment_runner.go b/src/cloud-api-adaptor/test/e2e/assessment_runner.go
@@ -48,6 +48,8 @@ type ExtraPod struct {
 	testCommands         []TestCommand
 }
 
+var testCase_secureComms_isActive bool
+
 type TestCase struct {
 	testing                     *testing.T
 	testEnv                     env.Environment
@@ -72,6 +74,7 @@ type TestCase struct {
 	expectedInstanceType        string
 	isNydusSnapshotter          bool
 	alternateImageName          string
+	secureCommsIsActive         bool
 }
 
 func (tc *TestCase) WithConfigMap(configMap *v1.ConfigMap) *TestCase {
@@ -134,6 +137,11 @@ func (tc *TestCase) WithAlternateImage(alternateImageName string) *TestCase {
 	return tc
 }
 
+func (tc *TestCase) WithSecureCommsIsActive() *TestCase {
+	tc.secureCommsIsActive = true
+	return tc
+}
+
 func (pod *ExtraPod) WithTestCommands(TestCommands []TestCommand) *ExtraPod {
 	pod.testCommands = TestCommands
 	return pod
@@ -420,6 +428,13 @@ func (tc *TestCase) Run() {
 						t.Errorf("VerifyAlternateImage failed: %v", err)
 					}
 				}
+
+				if tc.secureCommsIsActive {
+					err := VerifySecureCommsActivated(ctx, t, client, tc.pod)
+					if err != nil {
+						t.Errorf("VerifySecureCommsActivated failed: %v", err)
+					}
+				}
 			}
 
 			if tc.extraPods != nil {

diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go
@@ -35,6 +35,15 @@ func DoTestCreateSimplePod(t *testing.T, e env.Environment, assert CloudAssert)
 	}
 }
 
+func DoTestLibvirtCreateSimplePodWithSecureCommsIsValid(t *testing.T, e env.Environment, assert CloudAssert) {
+	//if !testCase_secureComms_isActive {
+	if os.Getenv("XYZ") != "xyz" {
+		t.Skip("Skip - SecureComms is configured to be inactive - no need to test")
+	}
+	pod := NewBusyboxPodWithName(E2eNamespace, "simple-test-with-security-comms-is-active").GetPodOrFatal(t)
+	NewTestCase(t, e, "SimplePeerPodWithSecureComms", assert, "PodVM is created with secure comms").WithPod(pod).WithSecureCommsIsActive().Run()
+}
+
 func DoTestDeleteSimplePod(t *testing.T, e env.Environment, assert CloudAssert) {
 	pod := NewBusyboxPodWithName(E2eNamespace, "deletion-test").GetPodOrFatal(t)
 	duration := assert.DefaultTimeout()

diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go
@@ -18,6 +18,11 @@ func TestLibvirtCreateSimplePod(t *testing.T) {
 	DoTestCreateSimplePod(t, testEnv, assert)
 }
 
+func TestLibvirtCreateSimplePodWithSecureCommsIsValid(t *testing.T) {
+	assert := LibvirtAssert{}
+	DoTestLibvirtCreateSimplePodWithSecureCommsIsValid(t, testEnv, assert)
+}
+
 func TestLibvirtCreatePodWithConfigMap(t *testing.T) {
 	assert := LibvirtAssert{}
 	DoTestCreatePodWithConfigMap(t, testEnv, assert)

diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go
@@ -123,6 +123,11 @@ func TestMain(m *testing.M) {
 
 		// Get properties
 		props := provisioner.GetProperties(ctx, cfg)
+		if props["SECURE_COMMS"] == "true" {
+			os.Setenv("XYZ", "xyz")
+			//testCase_secureComms_isActive = true
+			log.Info("Do setup secureComms is active")
+		}
 
 		// Set CONTAINER_RUNTIME env variable if present in the properties
 		// Default value is containerd.

diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go
@@ -23,17 +23,21 @@ const AlternateVolumeName = "another-podvm-base.qcow2"
 
 // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt.
 type LibvirtProvisioner struct {
-	conn             *libvirt.Connect // Libvirt connection
-	containerRuntime string           // Name of the container runtime
-	network          string           // Network name
-	ssh_key_file     string           // SSH key file used to connect to Libvirt
-	storage          string           // Storage pool name
-	uri              string           // Libvirt URI
-	wd               string           // libvirt's directory path on this repository
-	volumeName       string           // Podvm volume name
-	clusterName      string           // Cluster name
-	tunnelType       string           // Tunnel Type
-	vxlanPort        string           // VXLAN port number
+	conn                    *libvirt.Connect // Libvirt connection
+	containerRuntime        string           // Name of the container runtime
+	network                 string           // Network name
+	ssh_key_file            string           // SSH key file used to connect to Libvirt
+	storage                 string           // Storage pool name
+	uri                     string           // Libvirt URI
+	wd                      string           // libvirt's directory path on this repository
+	volumeName              string           // Podvm volume name
+	clusterName             string           // Cluster name
+	tunnelType              string           // Tunnel Type
+	vxlanPort               string           // VXLAN port number
+	secure_comms            string           // Activate CAA SECURE_COMMS
+	secure_comms_no_trustee string           // Deactivate Trustee mode in SECURE_COMMS
+	secure_comms_kbs_addr   string           // KBS URL
+	initdata                string           // InitData
 }
 
 // LibvirtInstallOverlay implements the InstallOverlay interface
@@ -95,19 +99,43 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e
 		vxlanPort = properties["vxlan_port"]
 	}
 
+	secure_comms := "false"
+	if properties["SECURE_COMMS"] != "" {
+		secure_comms = properties["SECURE_COMMS"]
+	}
+
+	secure_comms_kbs_addr := ""
+	if properties["SECURE_COMMS_KBS_ADDR"] != "" {
+		secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"]
+	}
+
+	secure_comms_no_trustee := "false"
+	if properties["SECURE_COMMS_NO_TRUSTEE"] != "" {
+		secure_comms_no_trustee = properties["SECURE_COMMS_NO_TRUSTEE"]
+	}
+
+	initdata := ""
+	if properties["INITDATA"] != "" {
+		initdata = properties["INITDATA"]
+	}
+
 	// TODO: Check network and storage are not nil?
 	return &LibvirtProvisioner{
-		conn:             conn,
-		containerRuntime: properties["container_runtime"],
-		network:          network,
-		ssh_key_file:     ssh_key_file,
-		storage:          storage,
-		uri:              uri,
-		wd:               wd,
-		volumeName:       vol_name,
-		clusterName:      clusterName,
-		tunnelType:       tunnelType,
-		vxlanPort:        vxlanPort,
+		conn:                    conn,
+		containerRuntime:        properties["container_runtime"],
+		network:                 network,
+		ssh_key_file:            ssh_key_file,
+		storage:                 storage,
+		uri:                     uri,
+		wd:                      wd,
+		volumeName:              vol_name,
+		clusterName:             clusterName,
+		tunnelType:              tunnelType,
+		vxlanPort:               vxlanPort,
+		secure_comms:            secure_comms,
+		secure_comms_kbs_addr:   secure_comms_kbs_addr,
+		secure_comms_no_trustee: secure_comms_no_trustee,
+		initdata:                initdata,
 	}, nil
 }
 
@@ -212,14 +240,18 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config)
 
 func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string {
 	return map[string]string{
-		"CONTAINER_RUNTIME": l.containerRuntime,
-		"network":           l.network,
-		"podvm_volume":      l.volumeName,
-		"ssh_key_file":      l.ssh_key_file,
-		"storage":           l.storage,
-		"uri":               l.uri,
-		"tunnel_type":       l.tunnelType,
-		"vxlan_port":        l.vxlanPort,
+		"CONTAINER_RUNTIME":       l.containerRuntime,
+		"network":                 l.network,
+		"podvm_volume":            l.volumeName,
+		"ssh_key_file":            l.ssh_key_file,
+		"storage":                 l.storage,
+		"uri":                     l.uri,
+		"tunnel_type":             l.tunnelType,
+		"vxlan_port":              l.vxlanPort,
+		"SECURE_COMMS":            l.secure_comms,
+		"SECURE_COMMS_KBS_ADDR":   l.secure_comms_kbs_addr,
+		"SECURE_COMMS_NO_TRUSTEE": l.secure_comms_no_trustee,
+		"INITDATA":                l.initdata,
 	}
 }
 
@@ -326,14 +358,17 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config,
 
 	// Mapping the internal properties to ConfigMapGenerator properties and their default values.
 	mapProps := map[string][2]string{
-		"network":      {"default", "LIBVIRT_NET"},
-		"storage":      {"default", "LIBVIRT_POOL"},
-		"pause_image":  {"", "PAUSE_IMAGE"},
-		"podvm_volume": {"", "LIBVIRT_VOL_NAME"},
-		"uri":          {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
-		"tunnel_type":  {"", "TUNNEL_TYPE"},
-		"vxlan_port":   {"", "VXLAN_PORT"},
-		"INITDATA":     {"", "INITDATA"},
+		"network":                 {"default", "LIBVIRT_NET"},
+		"storage":                 {"default", "LIBVIRT_POOL"},
+		"pause_image":             {"", "PAUSE_IMAGE"},
+		"podvm_volume":            {"", "LIBVIRT_VOL_NAME"},
+		"uri":                     {"qemu+ssh://[email protected]/system?no_verify=1", "LIBVIRT_URI"},
+		"tunnel_type":             {"", "TUNNEL_TYPE"},
+		"vxlan_port":              {"", "VXLAN_PORT"},
+		"INITDATA":                {"", "INITDATA"},
+		"SECURE_COMMS":            {"", "SECURE_COMMS"},
+		"SECURE_COMMS_NO_TRUSTEE": {"", "SECURE_COMMS_NO_TRUSTEE"},
+		"SECURE_COMMS_KBS_ADDR":   {"", "SECURE_COMMS_KBS_ADDR"},
 	}
 
 	for k, v := range mapProps {

diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go
@@ -220,7 +220,6 @@ func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error
 		wait.WithTimeout(time.Minute*1)); err != nil {
 		return err
 	}
-
 	return nil
 }
 
@@ -295,7 +294,15 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props
 		}
 	}
 
-	fmt.Printf("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName())
+	cmd = exec.Command("kubectl", "get", "cm", "peer-pods-cm", "-n", "confidential-containers-system", "-o", "yaml")
+	cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile()))
+	stdoutStderr, err = cmd.CombinedOutput()
+	log.Tracef("%v, output: %s", cmd, stdoutStderr)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName())
 	if err = wait.For(conditions.New(resources).ResourcesFound(&nodev1.RuntimeClassList{Items: []nodev1.RuntimeClass{*p.runtimeClass}}),
 		wait.WithTimeout(time.Second*60)); err != nil {
 		return err