add cache volume to the sidecar container

cmd/sidecar_mounter/main.go

@@ -31,6 +31,7 @@ import (
 
 	sidecarmounter "github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/sidecar_mounter"
 	"github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/util"
+	"github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/webhook"
 	"k8s.io/klog/v2"
 )
 
@@ -156,12 +157,11 @@ func main() {
 // 4. Mount options passing to gcsfuse (passed by the csi mounter).
 func prepareMountConfig(sp string) (*sidecarmounter.MountConfig, error) {
 	// socket path pattern: /gcsfuse-tmp/.volumes/<volume-name>/socket
-	dir := filepath.Dir(sp)
-	volumeName := filepath.Base(dir)
+	volumeName := filepath.Base(filepath.Dir(sp))
 	mc := sidecarmounter.MountConfig{
 		VolumeName: volumeName,
-		TempDir:    filepath.Join(dir, "temp-dir"),
-		ConfigFile: filepath.Join(dir, "config.yaml"),
+		CacheDir:   filepath.Join(webhook.SidecarContainerCacheVolumeMountPath, ".volumes", volumeName),
+		ConfigFile: filepath.Join(webhook.SidecarContainerTmpVolumeMountPath, ".volumes", volumeName, "config.yaml"),
 	}
 
 	klog.Infof("connecting to socket %q", sp)

pkg/cloud_provider/clientset/fake.go

@@ -40,9 +40,7 @@ func (c *FakeClientset) GetPod(_ context.Context, namespace, name string) (*v1.P
 			Containers: []v1.Container{
 				webhook.GetSidecarContainerSpec(config),
 			},
-			Volumes: []v1.Volume{
-				webhook.GetSidecarContainerVolumeSpec(),
-			},
+			Volumes: webhook.GetSidecarContainerVolumeSpec(),
 		},
 	}
 

pkg/sidecar_mounter/sidecar_mounter.go

@@ -50,7 +50,7 @@ type MountConfig struct {
 	FileDescriptor int       `json:"-"`
 	VolumeName     string    `json:"volumeName,omitempty"`
 	BucketName     string    `json:"bucketName,omitempty"`
-	TempDir        string    `json:"-"`
+	CacheDir       string    `json:"-"`
 	ConfigFile     string    `json:"-"`
 	Options        []string  `json:"options,omitempty"`
 	ErrWriter      io.Writer `json:"-"`
@@ -59,8 +59,8 @@ type MountConfig struct {
 func (m *Mounter) Mount(mc *MountConfig) (*exec.Cmd, error) {
 	klog.Infof("start to mount bucket %q for volume %q", mc.BucketName, mc.VolumeName)
 
-	if err := os.MkdirAll(mc.TempDir, os.ModePerm); err != nil {
-		return nil, fmt.Errorf("failed to create temp dir %q: %w", mc.TempDir, err)
+	if err := os.MkdirAll(mc.CacheDir+"/temp-dir", os.ModePerm); err != nil {
+		return nil, fmt.Errorf("failed to create temp dir %q: %w", mc.CacheDir+"/temp-dir", err)
 	}
 
 	flagMap, configFileFlagMap := mc.prepareMountArgs()
@@ -130,7 +130,7 @@ var boolFlags = map[string]bool{
 func (mc *MountConfig) prepareMountArgs() (map[string]string, map[string]string) {
 	flagMap := map[string]string{
 		"app-name":    GCSFuseAppName,
-		"temp-dir":    mc.TempDir,
+		"temp-dir":    mc.CacheDir + "/temp-dir",
 		"config-file": mc.ConfigFile,
 		"foreground":  "",
 		"uid":         "0",

pkg/sidecar_mounter/sidecar_mounter_test.go

@@ -28,7 +28,7 @@ import (
 var (
 	defaultFlagMap = map[string]string{
 		"app-name":    GCSFuseAppName,
-		"temp-dir":    "test-temp-dir",
+		"temp-dir":    "test-cache-dir/temp-dir",
 		"config-file": "test-config-file",
 		"foreground":  "",
 		"uid":         "0",
@@ -67,7 +67,7 @@ func TestPrepareMountArgs(t *testing.T) {
 			name: "should return valid args correctly",
 			mc: &MountConfig{
 				BucketName: "test-bucket",
-				TempDir:    "test-temp-dir",
+				CacheDir:   "test-cache-dir",
 				ConfigFile: "test-config-file",
 			},
 			expectedArgs:          defaultFlagMap,
@@ -77,14 +77,14 @@ func TestPrepareMountArgs(t *testing.T) {
 			name: "should return valid args with options correctly",
 			mc: &MountConfig{
 				BucketName: "test-bucket",
-				TempDir:    "test-temp-dir",
+				CacheDir:   "test-cache-dir",
 				ConfigFile: "test-config-file",
 				Options:    []string{"uid=100", "gid=200", "debug_gcs", "max-conns-per-host=10", "implicit-dirs", "write:create-empty-file:false", "logging:severity:error"},
 			},
 			expectedArgs: map[string]string{
 				"implicit-dirs":      "",
 				"app-name":           GCSFuseAppName,
-				"temp-dir":           "test-temp-dir",
+				"temp-dir":           "test-cache-dir/temp-dir",
 				"config-file":        "test-config-file",
 				"foreground":         "",
 				"uid":                "100",
@@ -103,14 +103,14 @@ func TestPrepareMountArgs(t *testing.T) {
 			name: "should return valid args with bool options correctly",
 			mc: &MountConfig{
 				BucketName: "test-bucket",
-				TempDir:    "test-temp-dir",
+				CacheDir:   "test-cache-dir",
 				ConfigFile: "test-config-file",
 				Options:    []string{"uid=100", "gid=200", "debug_gcs", "max-conns-per-host=10", "implicit-dirs=true"},
 			},
 			expectedArgs: map[string]string{
 				"implicit-dirs=true": "",
 				"app-name":           GCSFuseAppName,
-				"temp-dir":           "test-temp-dir",
+				"temp-dir":           "test-cache-dir/temp-dir",
 				"config-file":        "test-config-file",
 				"foreground":         "",
 				"uid":                "100",
@@ -124,7 +124,7 @@ func TestPrepareMountArgs(t *testing.T) {
 			name: "should return valid args with error correctly",
 			mc: &MountConfig{
 				BucketName: "test-bucket",
-				TempDir:    "test-temp-dir",
+				CacheDir:   "test-cache-dir",
 				ConfigFile: "test-config-file",
 				Options:    invalidArgs,
 			},
@@ -135,13 +135,13 @@ func TestPrepareMountArgs(t *testing.T) {
 			name: "should return valid args with custom app-name",
 			mc: &MountConfig{
 				BucketName: "test-bucket",
-				TempDir:    "test-temp-dir",
+				CacheDir:   "test-cache-dir",
 				ConfigFile: "test-config-file",
 				Options:    []string{"app-name=Vertex"},
 			},
 			expectedArgs: map[string]string{
 				"app-name":    GCSFuseAppName + "-Vertex",
-				"temp-dir":    "test-temp-dir",
+				"temp-dir":    "test-cache-dir/temp-dir",
 				"config-file": "test-config-file",
 				"foreground":  "",
 				"uid":         "0",

pkg/util/util.go

@@ -138,7 +138,7 @@ func PrepareEmptyDir(targetPath string, createEmptyDir bool) (string, error) {
 	}
 
 	r := regexp.MustCompile(`kubernetes\.io~csi/(.*)/mount`)
-	emptyDirBasePath := r.ReplaceAllString(targetPath, fmt.Sprintf("kubernetes.io~empty-dir/%v/.volumes/$1", webhook.SidecarContainerVolumeName))
+	emptyDirBasePath := r.ReplaceAllString(targetPath, fmt.Sprintf("kubernetes.io~empty-dir/%v/.volumes/$1", webhook.SidecarContainerTmpVolumeName))
 
 	if createEmptyDir {
 		if err := os.MkdirAll(emptyDirBasePath, 0o750); err != nil {

pkg/util/util_test.go

@@ -318,7 +318,7 @@ func TestPrepareEmptyDir(t *testing.T) {
 		{
 			name:                     "should return emptyDir path correctly",
 			targetPath:               "/var/lib/kubelet/pods/d2013878-3d56-45f9-89ec-0826612c89b6/volumes/kubernetes.io~csi/test-volume/mount",
-			expectedEmptyDirBasePath: fmt.Sprintf("/var/lib/kubelet/pods/d2013878-3d56-45f9-89ec-0826612c89b6/volumes/kubernetes.io~empty-dir/%v/.volumes/test-volume", webhook.SidecarContainerVolumeName),
+			expectedEmptyDirBasePath: fmt.Sprintf("/var/lib/kubelet/pods/d2013878-3d56-45f9-89ec-0826612c89b6/volumes/kubernetes.io~empty-dir/%v/.volumes/test-volume", webhook.SidecarContainerTmpVolumeName),
 			expectedError:            false,
 		},
 		{

pkg/webhook/mutatingwebhook.go

@@ -110,7 +110,7 @@ func (si *SidecarInjector) Handle(_ context.Context, req admission.Request) admi
 	klog.Infof("mutating Pod: Name %q, GenerateName %q, Namespace %q, CPU limit %q, memory limit %q, ephemeral storage limit %q", pod.Name, pod.GenerateName, pod.Namespace, configCopy.CPULimit.String(), configCopy.MemoryLimit.String(), configCopy.EphemeralStorageLimit.String())
 	// the gcsfuse sidecar container has to before the containers that consume the gcsfuse volume
 	pod.Spec.Containers = append([]corev1.Container{GetSidecarContainerSpec(configCopy)}, pod.Spec.Containers...)
-	pod.Spec.Volumes = append([]corev1.Volume{GetSidecarContainerVolumeSpec()}, pod.Spec.Volumes...)
+	pod.Spec.Volumes = append(GetSidecarContainerVolumeSpec(), pod.Spec.Volumes...)
 	marshaledPod, err := json.Marshal(pod)
 	if err != nil {
 		return admission.Errored(http.StatusBadRequest, fmt.Errorf("failed to marshal pod: %w", err))

pkg/webhook/sidecar_spec.go

@@ -25,9 +25,11 @@ import (
 )
 
 const (
-	SidecarContainerName            = "gke-gcsfuse-sidecar"
-	SidecarContainerVolumeName      = "gke-gcsfuse-tmp"
-	SidecarContainerVolumeMountPath = "/gcsfuse-tmp"
+	SidecarContainerName                 = "gke-gcsfuse-sidecar"
+	SidecarContainerTmpVolumeName        = "gke-gcsfuse-tmp"
+	SidecarContainerTmpVolumeMountPath   = "/gcsfuse-tmp"
+	SidecarContainerCacheVolumeName      = "gke-gcsfuse-cache"
+	SidecarContainerCacheVolumeMountPath = "/gcsfuse-cache"
 
 	// See the nonroot user discussion: https://github.com/GoogleContainerTools/distroless/issues/443
 	NobodyUID = 65534
@@ -69,53 +71,82 @@ func GetSidecarContainerSpec(c *Config) v1.Container {
 		},
 		VolumeMounts: []v1.VolumeMount{
 			{
-				Name:      SidecarContainerVolumeName,
-				MountPath: SidecarContainerVolumeMountPath,
+				Name:      SidecarContainerTmpVolumeName,
+				MountPath: SidecarContainerTmpVolumeMountPath,
+			},
+			{
+				Name:      SidecarContainerCacheVolumeName,
+				MountPath: SidecarContainerCacheVolumeMountPath,
 			},
 		},
 	}
 }
 
-func GetSidecarContainerVolumeSpec() v1.Volume {
-	return v1.Volume{
-		Name: SidecarContainerVolumeName,
-		VolumeSource: v1.VolumeSource{
-			EmptyDir: &v1.EmptyDirVolumeSource{},
+func GetSidecarContainerVolumeSpec() []v1.Volume {
+	return []v1.Volume{
+		{
+			Name: SidecarContainerTmpVolumeName,
+			VolumeSource: v1.VolumeSource{
+				EmptyDir: &v1.EmptyDirVolumeSource{},
+			},
+		},
+		{
+			Name: SidecarContainerCacheVolumeName,
+			VolumeSource: v1.VolumeSource{
+				EmptyDir: &v1.EmptyDirVolumeSource{},
+			},
 		},
 	}
 }
 
 // ValidatePodHasSidecarContainerInjected validates the following:
 // 1. One of the container name matches the sidecar container name.
 // 2. The image name matches.
-// 3. The container has a volume with the sidecar container volume name.
-// 4. The volume has the sidecar container volume mount path.
-// 5. The Pod has an emptyDir volume with the sidecar container volume name.
+// 3. The container uses the temp volume and cache volume.
+// 4. The temp volume and cache volume have correct volume mount paths.
+// 5. The Pod has the temp volume and cache volume. The temp volume has to be an emptyDir.
 func ValidatePodHasSidecarContainerInjected(image string, pod *v1.Pod) bool {
 	containerInjected := false
-	volumeInjected := false
+	tempVolumeInjected := false
+	cacheVolumeInjected := false
 	expectedImageWithoutTag := strings.Split(image, ":")[0]
 	for _, c := range pod.Spec.Containers {
-		if c.Name == SidecarContainerName && strings.Split(c.Image, ":")[0] == expectedImageWithoutTag {
+		if c.Name == SidecarContainerName {
+			if strings.Split(c.Image, ":")[0] == expectedImageWithoutTag &&
+				c.SecurityContext != nil &&
+				*c.SecurityContext.RunAsUser == NobodyUID &&
+				*c.SecurityContext.RunAsGroup == NobodyGID {
+				containerInjected = true
+			}
+
 			for _, v := range c.VolumeMounts {
-				if v.Name == SidecarContainerVolumeName && v.MountPath == SidecarContainerVolumeMountPath {
-					containerInjected = true
+				if v.Name == SidecarContainerTmpVolumeName && v.MountPath == SidecarContainerTmpVolumeMountPath {
+					tempVolumeInjected = true
+				}
 
-					break
+				if v.Name == SidecarContainerCacheVolumeName && v.MountPath == SidecarContainerCacheVolumeMountPath {
+					cacheVolumeInjected = true
 				}
 			}
 
 			break
 		}
 	}
 
-	for _, v := range pod.Spec.Volumes {
-		if v.Name == SidecarContainerVolumeName && v.VolumeSource.EmptyDir != nil {
-			volumeInjected = true
+	if !containerInjected || !tempVolumeInjected || !cacheVolumeInjected {
+		return false
+	}
 
-			break
+	tempVolumeInjected, cacheVolumeInjected = false, false
+
+	for _, v := range pod.Spec.Volumes {
+		if v.Name == SidecarContainerTmpVolumeName && v.VolumeSource.EmptyDir != nil {
+			tempVolumeInjected = true
+		}
+		if v.Name == SidecarContainerCacheVolumeName {
+			cacheVolumeInjected = true
 		}
 	}
 
-	return containerInjected && volumeInjected
+	return containerInjected && tempVolumeInjected && cacheVolumeInjected
 }