From 81af629f384bf0828fb611930d9374bbd23cd680 Mon Sep 17 00:00:00 2001 From: David Hadas Date: Sat, 21 Sep 2024 13:42:41 -0500 Subject: [PATCH] SecureComms: E2e test SecureComms without KBS Add support for e2e testing SecureComms without KBS Signed-off-by: David Hadas --- .../libvirt/config_libvirt.sh | 11 +- .../libvirt/e2e_matrix_libvirt.json | 2 +- .../test/e2e/assessment_helpers.go | 15 +++ .../test/e2e/assessment_runner.go | 15 +++ .../test/e2e/common_suite.go | 9 ++ .../test/e2e/libvirt_test.go | 5 + src/cloud-api-adaptor/test/e2e/main_test.go | 5 + .../provisioner/libvirt/provision_common.go | 111 ++++++++++++------ .../test/provisioner/provision.go | 11 +- 9 files changed, 141 insertions(+), 43 deletions(-) diff --git a/src/cloud-api-adaptor/libvirt/config_libvirt.sh b/src/cloud-api-adaptor/libvirt/config_libvirt.sh index 5ad41be22..ff1b560ed 100755 --- a/src/cloud-api-adaptor/libvirt/config_libvirt.sh +++ b/src/cloud-api-adaptor/libvirt/config_libvirt.sh @@ -92,7 +92,7 @@ installK8sclis() { fi } -TEST_E2E_SECURE_COMMS=${TEST_E2E_SECURE_COMMS:-none}. +TEST_E2E_SECURE_COMMS=${TEST_E2E_SECURE_COMMS:-none} echo "SECURE_COMMS is ${TEST_E2E_SECURE_COMMS}" echo "Installing Go..." @@ -123,7 +123,14 @@ echo "libvirt_ssh_key_file=\"id_rsa\"" >> libvirt.properties echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties # switch to the appropriate e2e test and add configs to libvirt.properties as needed -case $TEST_E2E_SECURE_COMMS in +case ${TEST_E2E_SECURE_COMMS} in + + withoutKbs) + echo "processing withoutKbs" + echo "SECURE_COMMS=\"true\"" >> libvirt.properties + echo "SECURE_COMMS_NO_TRUSTEE=\"true\"" >> libvirt.properties + echo "INITDATA=\"\"" >> libvirt.properties + ;; *) echo "processing none" diff --git a/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json b/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json index d0ed40b54..58238feff 100644 --- a/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json +++ b/src/cloud-api-adaptor/libvirt/e2e_matrix_libvirt.json @@ -1,6 +1,6 @@ { "container_runtime": ["containerd", "crio"], - "secure_comms": ["none"], + "secure_comms": ["none", "withoutKbs"], "os": ["ubuntu"], "provider": ["generic"], "arch": ["amd64"] diff --git a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go index 10976f332..2ea652c55 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_helpers.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_helpers.go @@ -292,6 +292,21 @@ func VerifyAlternateImage(ctx context.Context, t *testing.T, client klient.Clien return nil } +func VerifySecureCommsActivated(ctx context.Context, t *testing.T, client klient.Client, pod *v1.Pod) error { + nodeName, err := GetNodeNameFromPod(ctx, client, pod) + if err != nil { + return fmt.Errorf("VerifySecureCommsConnected: GetNodeNameFromPod failed with %v", err) + } + + expectedSuccessMessage := "Using PP SecureComms" + err = VerifyCaaPodLogContains(ctx, t, client, nodeName, expectedSuccessMessage) + if err != nil { + return fmt.Errorf("VerifySecureCommsConnected: failed: %v", err) + } + t.Logf("PodVM was brought up using SecureComms") + return nil +} + func VerifyCaaPodLogContains(ctx context.Context, t *testing.T, client klient.Client, nodeName, expected string) error { caaPod, err := getCaaPod(ctx, client, t, nodeName) if err != nil { diff --git a/src/cloud-api-adaptor/test/e2e/assessment_runner.go b/src/cloud-api-adaptor/test/e2e/assessment_runner.go index 15e75f723..1af006cb6 100644 --- a/src/cloud-api-adaptor/test/e2e/assessment_runner.go +++ b/src/cloud-api-adaptor/test/e2e/assessment_runner.go @@ -48,6 +48,8 @@ type ExtraPod struct { testCommands []TestCommand } +var testCase_secureComms_isActive bool + type TestCase struct { testing *testing.T testEnv env.Environment @@ -72,6 +74,7 @@ type TestCase struct { expectedInstanceType string isNydusSnapshotter bool alternateImageName string + secureCommsIsActive bool } func (tc *TestCase) WithConfigMap(configMap *v1.ConfigMap) *TestCase { @@ -134,6 +137,11 @@ func (tc *TestCase) WithAlternateImage(alternateImageName string) *TestCase { return tc } +func (tc *TestCase) WithSecureCommsIsActive() *TestCase { + tc.secureCommsIsActive = true + return tc +} + func (pod *ExtraPod) WithTestCommands(TestCommands []TestCommand) *ExtraPod { pod.testCommands = TestCommands return pod @@ -420,6 +428,13 @@ func (tc *TestCase) Run() { t.Errorf("VerifyAlternateImage failed: %v", err) } } + + if tc.secureCommsIsActive { + err := VerifySecureCommsActivated(ctx, t, client, tc.pod) + if err != nil { + t.Errorf("VerifySecureCommsActivated failed: %v", err) + } + } } if tc.extraPods != nil { diff --git a/src/cloud-api-adaptor/test/e2e/common_suite.go b/src/cloud-api-adaptor/test/e2e/common_suite.go index 9bce01655..2e9e21518 100644 --- a/src/cloud-api-adaptor/test/e2e/common_suite.go +++ b/src/cloud-api-adaptor/test/e2e/common_suite.go @@ -35,6 +35,15 @@ func DoTestCreateSimplePod(t *testing.T, e env.Environment, assert CloudAssert) } } +func DoTestLibvirtCreateSimplePodWithSecureCommsIsValid(t *testing.T, e env.Environment, assert CloudAssert) { + //if !testCase_secureComms_isActive { + if os.Getenv("XYZ") != "xyz" { + t.Skip("Skip - SecureComms is configured to be inactive - no need to test") + } + pod := NewBusyboxPodWithName(E2eNamespace, "simple-test-with-security-comms-is-active").GetPodOrFatal(t) + NewTestCase(t, e, "SimplePeerPodWithSecureComms", assert, "PodVM is created with secure comms").WithPod(pod).WithSecureCommsIsActive().Run() +} + func DoTestDeleteSimplePod(t *testing.T, e env.Environment, assert CloudAssert) { pod := NewBusyboxPodWithName(E2eNamespace, "deletion-test").GetPodOrFatal(t) duration := assert.DefaultTimeout() diff --git a/src/cloud-api-adaptor/test/e2e/libvirt_test.go b/src/cloud-api-adaptor/test/e2e/libvirt_test.go index 4a3b33b8f..46e726e5e 100644 --- a/src/cloud-api-adaptor/test/e2e/libvirt_test.go +++ b/src/cloud-api-adaptor/test/e2e/libvirt_test.go @@ -18,6 +18,11 @@ func TestLibvirtCreateSimplePod(t *testing.T) { DoTestCreateSimplePod(t, testEnv, assert) } +func TestLibvirtCreateSimplePodWithSecureCommsIsValid(t *testing.T) { + assert := LibvirtAssert{} + DoTestLibvirtCreateSimplePodWithSecureCommsIsValid(t, testEnv, assert) +} + func TestLibvirtCreatePodWithConfigMap(t *testing.T) { assert := LibvirtAssert{} DoTestCreatePodWithConfigMap(t, testEnv, assert) diff --git a/src/cloud-api-adaptor/test/e2e/main_test.go b/src/cloud-api-adaptor/test/e2e/main_test.go index 330b5d591..8ac8873c3 100644 --- a/src/cloud-api-adaptor/test/e2e/main_test.go +++ b/src/cloud-api-adaptor/test/e2e/main_test.go @@ -123,6 +123,11 @@ func TestMain(m *testing.M) { // Get properties props := provisioner.GetProperties(ctx, cfg) + if props["SECURE_COMMS"] == "true" { + os.Setenv("XYZ", "xyz") + //testCase_secureComms_isActive = true + log.Info("Do setup secureComms is active") + } // Set CONTAINER_RUNTIME env variable if present in the properties // Default value is containerd. diff --git a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go index 6cc9dd1c2..eed55669e 100644 --- a/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go +++ b/src/cloud-api-adaptor/test/provisioner/libvirt/provision_common.go @@ -23,17 +23,21 @@ const AlternateVolumeName = "another-podvm-base.qcow2" // LibvirtProvisioner implements the CloudProvisioner interface for Libvirt. type LibvirtProvisioner struct { - conn *libvirt.Connect // Libvirt connection - containerRuntime string // Name of the container runtime - network string // Network name - ssh_key_file string // SSH key file used to connect to Libvirt - storage string // Storage pool name - uri string // Libvirt URI - wd string // libvirt's directory path on this repository - volumeName string // Podvm volume name - clusterName string // Cluster name - tunnelType string // Tunnel Type - vxlanPort string // VXLAN port number + conn *libvirt.Connect // Libvirt connection + containerRuntime string // Name of the container runtime + network string // Network name + ssh_key_file string // SSH key file used to connect to Libvirt + storage string // Storage pool name + uri string // Libvirt URI + wd string // libvirt's directory path on this repository + volumeName string // Podvm volume name + clusterName string // Cluster name + tunnelType string // Tunnel Type + vxlanPort string // VXLAN port number + secure_comms string // Activate CAA SECURE_COMMS + secure_comms_no_trustee string // Deactivate Trustee mode in SECURE_COMMS + secure_comms_kbs_addr string // KBS URL + initdata string // InitData } // LibvirtInstallOverlay implements the InstallOverlay interface @@ -95,19 +99,43 @@ func NewLibvirtProvisioner(properties map[string]string) (pv.CloudProvisioner, e vxlanPort = properties["vxlan_port"] } + secure_comms := "false" + if properties["SECURE_COMMS"] != "" { + secure_comms = properties["SECURE_COMMS"] + } + + secure_comms_kbs_addr := "" + if properties["SECURE_COMMS_KBS_ADDR"] != "" { + secure_comms_kbs_addr = properties["SECURE_COMMS_KBS_ADDR"] + } + + secure_comms_no_trustee := "false" + if properties["SECURE_COMMS_NO_TRUSTEE"] != "" { + secure_comms_no_trustee = properties["SECURE_COMMS_NO_TRUSTEE"] + } + + initdata := "" + if properties["INITDATA"] != "" { + initdata = properties["INITDATA"] + } + // TODO: Check network and storage are not nil? return &LibvirtProvisioner{ - conn: conn, - containerRuntime: properties["container_runtime"], - network: network, - ssh_key_file: ssh_key_file, - storage: storage, - uri: uri, - wd: wd, - volumeName: vol_name, - clusterName: clusterName, - tunnelType: tunnelType, - vxlanPort: vxlanPort, + conn: conn, + containerRuntime: properties["container_runtime"], + network: network, + ssh_key_file: ssh_key_file, + storage: storage, + uri: uri, + wd: wd, + volumeName: vol_name, + clusterName: clusterName, + tunnelType: tunnelType, + vxlanPort: vxlanPort, + secure_comms: secure_comms, + secure_comms_kbs_addr: secure_comms_kbs_addr, + secure_comms_no_trustee: secure_comms_no_trustee, + initdata: initdata, }, nil } @@ -212,14 +240,18 @@ func (l *LibvirtProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) func (l *LibvirtProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string { return map[string]string{ - "CONTAINER_RUNTIME": l.containerRuntime, - "network": l.network, - "podvm_volume": l.volumeName, - "ssh_key_file": l.ssh_key_file, - "storage": l.storage, - "uri": l.uri, - "tunnel_type": l.tunnelType, - "vxlan_port": l.vxlanPort, + "CONTAINER_RUNTIME": l.containerRuntime, + "network": l.network, + "podvm_volume": l.volumeName, + "ssh_key_file": l.ssh_key_file, + "storage": l.storage, + "uri": l.uri, + "tunnel_type": l.tunnelType, + "vxlan_port": l.vxlanPort, + "SECURE_COMMS": l.secure_comms, + "SECURE_COMMS_KBS_ADDR": l.secure_comms_kbs_addr, + "SECURE_COMMS_NO_TRUSTEE": l.secure_comms_no_trustee, + "INITDATA": l.initdata, } } @@ -326,14 +358,17 @@ func (lio *LibvirtInstallOverlay) Edit(ctx context.Context, cfg *envconf.Config, // Mapping the internal properties to ConfigMapGenerator properties and their default values. mapProps := map[string][2]string{ - "network": {"default", "LIBVIRT_NET"}, - "storage": {"default", "LIBVIRT_POOL"}, - "pause_image": {"", "PAUSE_IMAGE"}, - "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, - "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, - "tunnel_type": {"", "TUNNEL_TYPE"}, - "vxlan_port": {"", "VXLAN_PORT"}, - "INITDATA": {"", "INITDATA"}, + "network": {"default", "LIBVIRT_NET"}, + "storage": {"default", "LIBVIRT_POOL"}, + "pause_image": {"", "PAUSE_IMAGE"}, + "podvm_volume": {"", "LIBVIRT_VOL_NAME"}, + "uri": {"qemu+ssh://root@192.168.122.1/system?no_verify=1", "LIBVIRT_URI"}, + "tunnel_type": {"", "TUNNEL_TYPE"}, + "vxlan_port": {"", "VXLAN_PORT"}, + "INITDATA": {"", "INITDATA"}, + "SECURE_COMMS": {"", "SECURE_COMMS"}, + "SECURE_COMMS_NO_TRUSTEE": {"", "SECURE_COMMS_NO_TRUSTEE"}, + "SECURE_COMMS_KBS_ADDR": {"", "SECURE_COMMS_KBS_ADDR"}, } for k, v := range mapProps { diff --git a/src/cloud-api-adaptor/test/provisioner/provision.go b/src/cloud-api-adaptor/test/provisioner/provision.go index 3c69b8080..1654f9aea 100644 --- a/src/cloud-api-adaptor/test/provisioner/provision.go +++ b/src/cloud-api-adaptor/test/provisioner/provision.go @@ -220,7 +220,6 @@ func (p *CloudAPIAdaptor) Delete(ctx context.Context, cfg *envconf.Config) error wait.WithTimeout(time.Minute*1)); err != nil { return err } - return nil } @@ -295,7 +294,15 @@ func (p *CloudAPIAdaptor) Deploy(ctx context.Context, cfg *envconf.Config, props } } - fmt.Printf("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName()) + cmd = exec.Command("kubectl", "get", "cm", "peer-pods-cm", "-n", "confidential-containers-system", "-o", "yaml") + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG="+cfg.KubeconfigFile())) + stdoutStderr, err = cmd.CombinedOutput() + log.Tracef("%v, output: %s", cmd, stdoutStderr) + if err != nil { + return err + } + + log.Infof("Wait for the %s runtimeclass be created\n", p.runtimeClass.GetName()) if err = wait.For(conditions.New(resources).ResourcesFound(&nodev1.RuntimeClassList{Items: []nodev1.RuntimeClass{*p.runtimeClass}}), wait.WithTimeout(time.Second*60)); err != nil { return err