caa: add scratch-space logic

This introduces a new CAA parameter that allows to specify the disk size. If we have a disk size that is larger than 0 a cloud provider can attempt to consider this size to create space for an encrypted scratch partition that can be mounted to /run/container. Signed-off-by: Magnus Kulke <[email protected]>
confidential-containers · Dec 17, 2024 · 79c5f33 · 79c5f33
1 parent 7223c69
commit 79c5f33
Show file tree

Hide file tree

Showing 13 changed files with 289 additions and 166 deletions.
diff --git a/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go b/src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go
@@ -125,6 +125,7 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.StringVar(&cfg.serverConfig.Initdata, "initdata", "", "Default initdata for all Pods")
 		flags.BoolVar(&cfg.serverConfig.EnableCloudConfigVerify, "cloud-config-verify", false, "Enable cloud config verify - should use it for production")
 		flags.IntVar(&cfg.serverConfig.PeerPodsLimitPerNode, "peerpods-limit-per-node", 10, "peer pods limit per node (default=10)")
+		flags.Uint64Var(&cfg.serverConfig.DiskSize, "disk-size", 0, "Disk size in GB. Default is 0, which implies the default image disk size")
 
 		cloud.ParseCmd(flags)
 	})

diff --git a/src/cloud-api-adaptor/entrypoint.sh b/src/cloud-api-adaptor/entrypoint.sh
@@ -30,6 +30,7 @@ optionals+=""
 [[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} "
 [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
 [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
+[[ "${DISK_SIZE}" ]] && optionals+="-disk-size ${DISK_SIZE} "
 
 test_vars() {
     for i in "$@"; do

diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go
@@ -64,6 +64,8 @@ type ServerConfig struct {
 	SecureCommsPpOutbounds  string
 	SecureCommsKbsAddress   string
 	PeerPodsLimitPerNode    int
+	UseEncryptedDisk        bool
+	DiskSize                uint64
 }
 
 var logger = log.New(log.Writer(), "[adaptor/cloud] ", log.LstdFlags|log.Lmsgprefix)
@@ -222,17 +224,20 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 	instanceType := util.GetInstanceTypeFromAnnotation(req.Annotations)
 
 	// Get Pod VM cpu and memory from annotations
-	vcpus, memory, gpus := util.GetPodvmResourcesFromAnnotation(req.Annotations)
+	resources := util.GetPodVMResourcesFromAnnotation(req.Annotations)
+
+	// Add disk size to resources if set
+	if s.serverConfig.DiskSize > 0 {
+		resources.Storage = int64(s.serverConfig.DiskSize)
+	}
 
 	// Get Pod VM image from annotations
 	image := util.GetImageFromAnnotation(req.Annotations)
 
 	// Pod VM spec
 	vmSpec := provider.InstanceTypeSpec{
 		InstanceType: instanceType,
-		VCPUs:        vcpus,
-		Memory:       memory,
-		GPUs:         gpus,
+		Resources:    resources,
 		Image:        image,
 	}
 
@@ -332,6 +337,14 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r
 		},
 	}
 
+	if s.serverConfig.DiskSize > 0 {
+		// Write an empty file to indicate that we want to use  available space as sandbox storage
+		cloudConfig.WriteFiles = append(cloudConfig.WriteFiles, cloudinit.WriteFile{
+			Path:    UseScratchPath,
+			Content: "",
+		})
+	}
+
 	if authJSON != nil {
 		if len(authJSON) > cloudinit.DefaultAuthfileLimit {
 			logger.Printf("Credentials file is too large to be included in cloud-config")

diff --git a/src/cloud-api-adaptor/pkg/paths/paths.go b/src/cloud-api-adaptor/pkg/paths/paths.go
@@ -8,4 +8,5 @@ const (
 	AgentCfgPath     = "/run/peerpod/agent-config.toml"
 	ForwarderCfgPath = "/run/peerpod/daemon.json"
 	UserDataPath     = "/run/media/cidata/user-data"
+	UseScratchPath   = "/run/peerpod/mount-scratch"
 )
diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go
@@ -37,7 +37,7 @@ const (
 )
 
 var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix)
-var WriteFilesList = []string{AACfgPath, CDHCfgPath, AgentCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath}
+var WriteFilesList = []string{AACfgPath, CDHCfgPath, AgentCfgPath, ForwarderCfgPath, AuthFilePath, InitDataPath, UseScratchPath}
 var InitdDataFilesList = []string{AACfgPath, CDHCfgPath, PolicyPath}
 
 type Config struct {

diff --git a/src/cloud-api-adaptor/pkg/util/cloud.go b/src/cloud-api-adaptor/pkg/util/cloud.go
@@ -5,6 +5,7 @@ import (
 	"strconv"
 	"strings"
 
+	provider "github.com/confidential-containers/cloud-api-adaptor/src/cloud-providers"
 	cri "github.com/containerd/containerd/pkg/cri/annotations"
 	hypannotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 )
@@ -44,48 +45,48 @@ func GetImageFromAnnotation(annotations map[string]string) string {
 }
 
 // Method to get vCPU, memory and gpus from annotations
-func GetPodvmResourcesFromAnnotation(annotations map[string]string) (int64, int64, int64) {
-
-	var vcpuInt, memoryInt, gpuInt int64
+func GetPodVMResourcesFromAnnotation(annotations map[string]string) provider.PodVMResources {
+	var vcpus, gpus, memory int64
 	var err error
 
 	vcpu, ok := annotations[hypannotations.DefaultVCPUs]
 	if ok {
-		vcpuInt, err = strconv.ParseInt(vcpu, 10, 64)
+		vcpus, err = strconv.ParseInt(vcpu, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting vcpu to int64. Defaulting to 0: %v\n", err)
-			vcpuInt = 0
+			vcpus = 0
 		}
 	} else {
-		vcpuInt = 0
+		vcpus = 0
 	}
 
-	memory, ok := annotations[hypannotations.DefaultMemory]
+	mem, ok := annotations[hypannotations.DefaultMemory]
 	if ok {
 		// Use strconv.ParseInt to convert string to int64
-		memoryInt, err = strconv.ParseInt(memory, 10, 64)
+		memory, err = strconv.ParseInt(mem, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting memory to int64. Defaulting to 0: %v\n", err)
-			memoryInt = 0
+			memory = 0
 		}
 
 	} else {
-		memoryInt = 0
+		memory = 0
 	}
 
 	gpu, ok := annotations[hypannotations.DefaultGPUs]
 	if ok {
-		gpuInt, err = strconv.ParseInt(gpu, 10, 64)
+		gpus, err = strconv.ParseInt(gpu, 10, 64)
 		if err != nil {
 			fmt.Printf("Error converting gpu to int64. Defaulting to 0: %v\n", err)
-			gpuInt = 0
+			gpus = 0
 		}
 	} else {
-		gpuInt = 0
+		gpus = 0
 	}
 
-	// Return vCPU, memory and GPU
-	return vcpuInt, memoryInt, gpuInt
+	storage := int64(0)
+
+	return provider.PodVMResources{VCPUs: vcpus, Memory: memory, GPUs: gpus, Storage: storage}
 }
 
 // Method to get initdata from annotation

diff --git a/src/cloud-api-adaptor/pkg/util/cloud_test.go b/src/cloud-api-adaptor/pkg/util/cloud_test.go
@@ -98,7 +98,8 @@ func TestGetPodvmResourcesFromAnnotation(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, got1, got2 := GetPodvmResourcesFromAnnotation(tt.args.annotations)
+			r := GetPodVMResourcesFromAnnotation(tt.args.annotations)
+			got, got1, got2 := r.VCPUs, r.Memory, r.GPUs
 			if got != tt.want {
 				t.Errorf("GetPodvmResourcesFromAnnotation() got = %v, want %v", got, tt.want)
 			}

diff --git a/src/cloud-providers/aws/provider.go b/src/cloud-providers/aws/provider.go
@@ -370,8 +370,10 @@ func (p *awsProvider) updateInstanceTypeSpecList() error {
 		if err != nil {
 			return err
 		}
+		resources := provider.NewPodVMResources(vcpus, memory)
+		resources.GPUs = gpuCount
 		instanceTypeSpecList = append(instanceTypeSpecList,
-			provider.InstanceTypeSpec{InstanceType: instanceType, VCPUs: vcpus, Memory: memory, GPUs: gpuCount})
+			provider.InstanceTypeSpec{InstanceType: instanceType, Resources: resources})
 	}
 
 	// Sort the instanceTypeSpecList and update the serviceConfig

diff --git a/src/cloud-providers/azure/provider.go b/src/cloud-providers/azure/provider.go
@@ -11,7 +11,6 @@ import (
 	"log"
 	"net/netip"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
 
@@ -35,24 +34,44 @@ const (
 type azureProvider struct {
 	azureClient   azcore.TokenCredential
 	serviceConfig *Config
+	sshKey        armcompute.SSHPublicKey
 }
 
 func NewProvider(config *Config) (provider.Provider, error) {
 
 	logger.Printf("azure config %+v", config.Redact())
 
-	// Clean the config.SSHKeyPath to avoid bad paths
-	config.SSHKeyPath = filepath.Clean(config.SSHKeyPath)
-
 	azureClient, err := NewAzureClient(*config)
 	if err != nil {
 		logger.Printf("creating azure client: %v", err)
 		return nil, err
 	}
 
+	// The podvm disk doesn't support sshd logins. keys can be baked
+	// into a debug-image. The ARM api mandates a pubkey, though.
+	sshPublicKeyPath := os.ExpandEnv(config.SSHKeyPath)
+	var pubkeyBytes []byte
+	if _, err := os.Stat(sshPublicKeyPath); err == nil {
+		pubkeyBytes, err = os.ReadFile(sshPublicKeyPath)
+		if err != nil {
+			err = fmt.Errorf("reading ssh public key file: %w", err)
+			logger.Printf("%v", err)
+			return nil, err
+		}
+	} else {
+		err = fmt.Errorf("ssh public key: %w", err)
+		logger.Printf("%v", err)
+		return nil, err
+	}
+	dummySSHKey := armcompute.SSHPublicKey{
+		Path:    to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", config.SSHUserName)),
+		KeyData: to.Ptr(string(pubkeyBytes)),
+	}
+
 	provider := &azureProvider{
 		azureClient:   azureClient,
 		serviceConfig: config,
+		sshKey:        dummySSHKey,
 	}
 
 	if err = provider.updateInstanceSizeSpecList(); err != nil {
@@ -218,28 +237,12 @@ func (p *azureProvider) CreateInstance(ctx context.Context, podName, sandboxID s
 	diskName := fmt.Sprintf("%s-disk", instanceName)
 	nicName := fmt.Sprintf("%s-net", instanceName)
 
-	// require ssh key for authentication on linux
-	sshPublicKeyPath := os.ExpandEnv(p.serviceConfig.SSHKeyPath)
-	var sshBytes []byte
-	if _, err := os.Stat(sshPublicKeyPath); err == nil {
-		sshBytes, err = os.ReadFile(sshPublicKeyPath)
-		if err != nil {
-			err = fmt.Errorf("reading ssh public key file: %w", err)
-			logger.Printf("%v", err)
-			return nil, err
-		}
-	} else {
-		err = fmt.Errorf("ssh public key: %w", err)
-		logger.Printf("%v", err)
-		return nil, err
-	}
-
 	if spec.Image != "" {
 		logger.Printf("Choosing %s from annotation as the Azure Image for the PodVM image", spec.Image)
 		p.serviceConfig.ImageId = spec.Image
 	}
 
-	vmParameters, err := p.getVMParameters(instanceSize, diskName, cloudConfigData, sshBytes, instanceName, nicName)
+	vmParameters, err := p.getVMParameters(instanceSize, diskName, cloudConfigData, instanceName, nicName, spec.Resources.Storage)
 	if err != nil {
 		return nil, err
 	}
@@ -350,7 +353,10 @@ func (p *azureProvider) updateInstanceSizeSpecList() error {
 		}
 		for _, vmSize := range nextResult.VirtualMachineSizeListResult.Value {
 			if util.Contains(instanceSizes, *vmSize.Name) {
-				instanceSizeSpecList = append(instanceSizeSpecList, provider.InstanceTypeSpec{InstanceType: *vmSize.Name, VCPUs: int64(*vmSize.NumberOfCores), Memory: int64(*vmSize.MemoryInMB)})
+				vcpus, memory := int64(*vmSize.NumberOfCores), int64(*vmSize.MemoryInMB)
+				resources := provider.NewPodVMResources(vcpus, memory)
+				instanceSizeSpec := provider.InstanceTypeSpec{InstanceType: *vmSize.Name, Resources: resources}
+				instanceSizeSpecList = append(instanceSizeSpecList, instanceSizeSpec)
 			}
 		}
 	}
@@ -371,7 +377,7 @@ func (p *azureProvider) getResourceTags() map[string]*string {
 	return tags
 }
 
-func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig string, sshBytes []byte, instanceName, nicName string) (*armcompute.VirtualMachine, error) {
+func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig, instanceName, nicName string, diskSize int64) (*armcompute.VirtualMachine, error) {
 	userDataB64 := base64.StdEncoding.EncodeToString([]byte(cloudConfig))
 
 	// Azure limits the base64 encrypted userData to 64KB.
@@ -416,6 +422,18 @@ func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig stri
 
 	networkConfig := p.buildNetworkConfig(nicName)
 
+	osDisk := armcompute.OSDisk{
+		Name:         to.Ptr(diskName),
+		CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage),
+		Caching:      to.Ptr(armcompute.CachingTypesReadWrite),
+		DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete),
+		ManagedDisk:  managedDiskParams,
+	}
+
+	if diskSize > 0 {
+		osDisk.DiskSizeGB = to.Ptr(int32(diskSize))
+	}
+
 	vmParameters := armcompute.VirtualMachine{
 		Location: to.Ptr(p.serviceConfig.Region),
 		Properties: &armcompute.VirtualMachineProperties{
@@ -424,25 +442,15 @@ func (p *azureProvider) getVMParameters(instanceSize, diskName, cloudConfig stri
 			},
 			StorageProfile: &armcompute.StorageProfile{
 				ImageReference: imgRef,
-				OSDisk: &armcompute.OSDisk{
-					Name:         to.Ptr(diskName),
-					CreateOption: to.Ptr(armcompute.DiskCreateOptionTypesFromImage),
-					Caching:      to.Ptr(armcompute.CachingTypesReadWrite),
-					DeleteOption: to.Ptr(armcompute.DiskDeleteOptionTypesDelete),
-					ManagedDisk:  managedDiskParams,
-				},
+				OSDisk:         &osDisk,
 			},
 			OSProfile: &armcompute.OSProfile{
 				AdminUsername: to.Ptr(p.serviceConfig.SSHUserName),
 				ComputerName:  to.Ptr(instanceName),
 				LinuxConfiguration: &armcompute.LinuxConfiguration{
 					DisablePasswordAuthentication: to.Ptr(true),
-					//TBD: replace with a suitable mechanism to use precreated SSH key
 					SSH: &armcompute.SSHConfiguration{
-						PublicKeys: []*armcompute.SSHPublicKey{{
-							Path:    to.Ptr(fmt.Sprintf("/home/%s/.ssh/authorized_keys", p.serviceConfig.SSHUserName)),
-							KeyData: to.Ptr(string(sshBytes)),
-						}},
+						PublicKeys: []*armcompute.SSHPublicKey{&p.sshKey},
 					},
 				},
 			},

diff --git a/src/cloud-providers/ibmcloud/provider.go b/src/cloud-providers/ibmcloud/provider.go
@@ -332,7 +332,8 @@ func (p *ibmcloudVPCProvider) updateInstanceProfileSpecList() error {
 		if err != nil {
 			return err
 		}
-		instanceProfileSpecList = append(instanceProfileSpecList, provider.InstanceTypeSpec{InstanceType: profileType, VCPUs: vcpus, Memory: memory, Arch: arch})
+		resources := provider.NewPodVMResources(vcpus, memory)
+		instanceProfileSpecList = append(instanceProfileSpecList, provider.InstanceTypeSpec{InstanceType: profileType, Resources: resources, Arch: arch})
 	}
 
 	// Sort the instanceProfileSpecList and update the serviceConfig

diff --git a/src/cloud-providers/types.go b/src/cloud-providers/types.go
@@ -60,11 +60,25 @@ type Instance struct {
 	IPs  []netip.Addr
 }
 
+type PodVMResources struct {
+	VCPUs   int64
+	Memory  int64
+	GPUs    int64
+	Storage int64
+}
+
+func NewPodVMResources(vcpus, memory int64) PodVMResources {
+	return PodVMResources{
+		VCPUs:   vcpus,
+		Memory:  memory,
+		GPUs:    0,
+		Storage: 0,
+	}
+}
+
 type InstanceTypeSpec struct {
 	InstanceType string
-	VCPUs        int64
-	Memory       int64
+	Resources    PodVMResources
 	Arch         string
-	GPUs         int64
 	Image        string
 }