Add metadata prefetch support.

Makefile

@@ -29,10 +29,12 @@ IDENTITY_PROVIDER ?= $(shell kubectl get --raw /.well-known/openid-configuration
 DRIVER_BINARY = gcs-fuse-csi-driver
 SIDECAR_BINARY = gcs-fuse-csi-driver-sidecar-mounter
 WEBHOOK_BINARY = gcs-fuse-csi-driver-webhook
+PREFETCH_BINARY = gcs-fuse-csi-driver-metadata-prefetch
 
 DRIVER_IMAGE = ${REGISTRY}/${DRIVER_BINARY}
 SIDECAR_IMAGE = ${REGISTRY}/${SIDECAR_BINARY}
 WEBHOOK_IMAGE = ${REGISTRY}/${WEBHOOK_BINARY}
+PREFETCH_IMAGE = ${REGISTRY}/${PREFETCH_BINARY}
 
 DOCKER_BUILDX_ARGS ?= --push --builder multiarch-multiplatform-builder --build-arg STAGINGVERSION=${STAGINGVERSION}
 ifneq ("$(shell docker buildx build --help | grep 'provenance')", "")
@@ -46,7 +48,7 @@ $(info DRIVER_IMAGE is ${DRIVER_IMAGE})
 $(info SIDECAR_IMAGE is ${SIDECAR_IMAGE})
 $(info WEBHOOK_IMAGE is ${WEBHOOK_IMAGE})
 
-all: driver sidecar-mounter webhook
+all: driver sidecar-mounter webhook metadata-prefetch
 
 driver:
 	mkdir -p ${BINDIR}
@@ -56,6 +58,10 @@ sidecar-mounter:
 	mkdir -p ${BINDIR}
 	CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -mod vendor -ldflags "${LDFLAGS}" -o ${BINDIR}/${SIDECAR_BINARY} cmd/sidecar_mounter/main.go
 
+metadata-prefetch:
+	mkdir -p ${BINDIR}
+	CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -mod vendor -ldflags "${LDFLAGS}" -o ${BINDIR}/${PREFETCH_BINARY} cmd/metadata_prefetch/main.go
+
 webhook:
 	mkdir -p ${BINDIR}
 	CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -mod vendor -ldflags "${LDFLAGS}" -o ${BINDIR}/${WEBHOOK_BINARY} cmd/webhook/main.go
@@ -129,18 +135,27 @@ ifeq (${BUILD_ARM}, true)
 	make build-image-linux-arm64
 	docker manifest create ${DRIVER_IMAGE}:${STAGINGVERSION} ${DRIVER_IMAGE}:${STAGINGVERSION}_linux_amd64 ${DRIVER_IMAGE}:${STAGINGVERSION}_linux_arm64
 	docker manifest create ${SIDECAR_IMAGE}:${STAGINGVERSION} ${SIDECAR_IMAGE}:${STAGINGVERSION}_linux_amd64 ${SIDECAR_IMAGE}:${STAGINGVERSION}_linux_arm64
+	docker manifest create ${PREFETCH_IMAGE}:${STAGINGVERSION} ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_amd64 ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_arm64
 else
 	docker manifest create ${DRIVER_IMAGE}:${STAGINGVERSION} ${DRIVER_IMAGE}:${STAGINGVERSION}_linux_amd64
 	docker manifest create ${SIDECAR_IMAGE}:${STAGINGVERSION} ${SIDECAR_IMAGE}:${STAGINGVERSION}_linux_amd64
+	docker manifest create ${PREFETCH_IMAGE}:${STAGINGVERSION} ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_amd64
 endif
 
 	docker manifest create ${WEBHOOK_IMAGE}:${STAGINGVERSION} ${WEBHOOK_IMAGE}:${STAGINGVERSION}_linux_amd64
 
 	docker manifest push --purge ${DRIVER_IMAGE}:${STAGINGVERSION}
 	docker manifest push --purge ${SIDECAR_IMAGE}:${STAGINGVERSION}
+	docker manifest push --purge ${PREFETCH_IMAGE}:${STAGINGVERSION}
 	docker manifest push --purge ${WEBHOOK_IMAGE}:${STAGINGVERSION}
 
 build-image-linux-amd64:
+	docker buildx build ${DOCKER_BUILDX_ARGS} \
+		--file ./cmd/metadata_prefetch/Dockerfile \
+		--tag ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_amd64 \
+		--platform linux/amd64 \
+		--build-arg TARGETPLATFORM=linux/amd64 .
+
 	docker buildx build \
 		--file ./cmd/csi_driver/Dockerfile \
 		--tag validation_linux_amd64 \
@@ -164,6 +179,12 @@ build-image-linux-amd64:
 		--platform linux/amd64 .
 
 build-image-linux-arm64:
+	docker buildx build ${DOCKER_BUILDX_ARGS} \
+		--file ./cmd/sidecar_metadata_cache_populator/Dockerfile \
+		--tag ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_arm64 \
+		--platform linux/arm64 \
+		--build-arg TARGETPLATFORM=linux/arm64 .
+
 	docker buildx build \
 		--file ./cmd/csi_driver/Dockerfile \
 		--tag validation_linux_arm64 \
@@ -198,6 +219,7 @@ generate-spec-yaml:
 	cd ./deploy/overlays/${OVERLAY}; ${BINDIR}/kustomize edit set image gke.gcr.io/gcs-fuse-csi-driver=${DRIVER_IMAGE}:${STAGINGVERSION};
 	cd ./deploy/overlays/${OVERLAY}; ${BINDIR}/kustomize edit set image gke.gcr.io/gcs-fuse-csi-driver-webhook=${WEBHOOK_IMAGE}:${STAGINGVERSION};
 	cd ./deploy/overlays/${OVERLAY}; ${BINDIR}/kustomize edit add configmap gcsfusecsi-image-config --behavior=merge --disableNameSuffixHash --from-literal=sidecar-image=${SIDECAR_IMAGE}:${STAGINGVERSION};
+	cd ./deploy/overlays/${OVERLAY}; ${BINDIR}/kustomize edit add configmap gcsfusecsi-image-config --behavior=merge --disableNameSuffixHash --from-literal=metadata-sidecar-image=${PREFETCH_IMAGE}:${STAGINGVERSION};
 	echo "[{\"op\": \"replace\",\"path\": \"/spec/tokenRequests/0/audience\",\"value\": \"${PROJECT}.svc.id.goog\"}]" > ./deploy/overlays/${OVERLAY}/project_patch_csi_driver.json
 	echo "[{\"op\": \"replace\",\"path\": \"/webhooks/0/clientConfig/caBundle\",\"value\": \"${CA_BUNDLE}\"}]" > ./deploy/overlays/${OVERLAY}/caBundle_patch_MutatingWebhookConfiguration.json
 	echo "[{\"op\": \"replace\",\"path\": \"/spec/template/spec/containers/0/env/1/value\",\"value\": \"${IDENTITY_PROVIDER}\"}]" > ./deploy/overlays/${OVERLAY}/identity_provider_patch_csi_node.json

cmd/metadata_prefetch/Dockerfile

@@ -0,0 +1,44 @@
+# Copyright 2018 The Kubernetes Authors.
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build sidecar-mounter go binary
+FROM golang:1.22.7 AS metadata-prefetch-builder
+
+ARG STAGINGVERSION
+
+WORKDIR /gcs-fuse-csi-driver
+ADD . .
+RUN make metadata-prefetch BINDIR=/bin
+
+# go/gke-releasing-policies#base-images
+FROM gke.gcr.io/debian-base:bookworm-v1.0.4-gke.2 AS debian
+
+# go/gke-releasing-policies#base-images
+FROM  gcr.io/distroless/base-debian12 
+ARG TARGETPLATFORM
+
+# Copy existing binaries.
+COPY --from=debian /bin/ls /bin/ls
+
+# Copy dependencies.
+COPY --from=debian /lib/x86_64-linux-gnu/libselinux.so.1 /lib/x86_64-linux-gnu/libselinux.so.1
+COPY --from=debian /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu/libc.so.6
+COPY --from=debian /lib/x86_64-linux-gnu/libpcre2-8.so.0 /lib/x86_64-linux-gnu/libpcre2-8.so.0
+COPY --from=debian /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 
+
+# Copy the built binaries
+COPY --from=metadata-prefetch-builder /bin/gcs-fuse-csi-driver-metadata-prefetch /gcs-fuse-csi-driver-metadata-prefetch
+
+ENTRYPOINT ["/gcs-fuse-csi-driver-metadata-prefetch"]

cmd/metadata_prefetch/main.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+Copyright 2024 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+
+	"k8s.io/klog/v2"
+)
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Parse()
+
+	// Create cancellable context to pass into exec.
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Handle SIGTERM signal.
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGTERM)
+
+	go func() {
+		<-sigs
+		klog.Info("Caught SIGTERM signal: Terminating...")
+		cancel()
+
+		os.Exit(0) // Exit gracefully
+	}()
+
+	// Start the "ls" command in the background.
+	// All our volumes are mounted under the /volumes/ directory.
+	cmd := exec.CommandContext(ctx, "ls", "-R", "/volumes/")
+	cmd.Stdout = nil // Connects file descriptor to the null device (os.DevNull).
+
+	// TODO(hime): We should research stratergies to parallelize ls execution and speed up cache population.
+	err := cmd.Start()
+	if err == nil {
+		klog.Info("Running ls on bucket(s)")
+		err = cmd.Wait()
+		if err != nil {
+			klog.Errorf("Error while executing ls command: %v", err)
+		} else {
+			klog.Info("Metadata prefetch complete.")
+		}
+	} else {
+		klog.Errorf("Error starting ls command: %v.", err)
+	}
+
+	klog.Info("Going to sleep...")
+
+	// Keep the process running.
+	select {}
+}

cmd/webhook/main.go

@@ -52,6 +52,7 @@ var (
 	ephemeralStorageRequest = flag.String("sidecar-ephemeral-storage-request", "5Gi", "The default ephemeral storage request for gcsfuse sidecar container.")
 	ephemeralStorageLimit   = flag.String("sidecar-ephemeral-storage-limit", "5Gi", "The default ephemeral storage limit for gcsfuse sidecar container.")
 	sidecarImage            = flag.String("sidecar-image", "", "The gcsfuse sidecar container image.")
+	metadataSidecarImage    = flag.String("metadata-sidecar-image", "", "The gcsfuse sidecar container image.")
 
 	// These are set at compile time.
 	webhookVersion = "unknown"
@@ -72,7 +73,7 @@ func main() {
 	klog.Infof("Running Google Cloud Storage FUSE CSI driver admission webhook version %v, sidecar container image %v", webhookVersion, *sidecarImage)
 
 	// Load webhook config
-	c := wh.LoadConfig(*sidecarImage, *imagePullPolicy, *cpuRequest, *cpuLimit, *memoryRequest, *memoryLimit, *ephemeralStorageRequest, *ephemeralStorageLimit)
+	c := wh.LoadConfig(*sidecarImage, *imagePullPolicy, *cpuRequest, *cpuLimit, *memoryRequest, *memoryLimit, *ephemeralStorageRequest, *ephemeralStorageLimit, *metadataSidecarImage)
 
 	// Load config for manager, informers, listers
 	kubeConfig := config.GetConfigOrDie()

deploy/base/node/node.yaml

@@ -133,3 +133,4 @@ metadata:
   name: gcsfusecsi-image-config
 data:
   sidecar-image: gke.gcr.io/gcs-fuse-csi-driver-sidecar-mounter
+  metadata-sidecar-image: gke.gcr.io/gcs-fuse-csi-driver-metadata-prefetch

deploy/base/webhook/deployment.yaml

@@ -54,6 +54,7 @@ spec:
             - --sidecar-ephemeral-storage-limit=0
             - --sidecar-ephemeral-storage-request=5Gi
             - --sidecar-image=$(SIDECAR_IMAGE)
+            - --metadata-sidecar-image=$(METADATA_SIDECAR_IMAGE)
             - --sidecar-image-pull-policy=$(SIDECAR_IMAGE_PULL_POLICY)
             - --cert-dir=/etc/tls-certs
             - --port=22030
@@ -66,6 +67,11 @@ spec:
                 configMapKeyRef:
                   name: gcsfusecsi-image-config
                   key: sidecar-image
+            - name: METADATA_SIDECAR_IMAGE
+              valueFrom:
+                configMapKeyRef:
+                  name: gcsfusecsi-image-config
+                  key: metadata-sidecar-image
           resources:
             limits:
               cpu: 200m

pkg/webhook/config.go

@@ -27,8 +27,9 @@ import (
 )
 
 type Config struct {
-	ContainerImage  string `json:"-"`
-	ImagePullPolicy string `json:"-"`
+	ContainerImage         string `json:"-"`
+	MetadataContainerImage string `json:"-"`
+	ImagePullPolicy        string `json:"-"`
 	//nolint:tagliatelle
 	CPURequest resource.Quantity `json:"gke-gcsfuse/cpu-request,omitempty"`
 	//nolint:tagliatelle
@@ -43,9 +44,10 @@ type Config struct {
 	EphemeralStorageLimit resource.Quantity `json:"gke-gcsfuse/ephemeral-storage-limit,omitempty"`
 }
 
-func LoadConfig(containerImage, imagePullPolicy, cpuRequest, cpuLimit, memoryRequest, memoryLimit, ephemeralStorageRequest, ephemeralStorageLimit string) *Config {
+func LoadConfig(containerImage, imagePullPolicy, cpuRequest, cpuLimit, memoryRequest, memoryLimit, ephemeralStorageRequest, ephemeralStorageLimit, metadataContainerImage string) *Config {
 	return &Config{
 		ContainerImage:          containerImage,
+		MetadataContainerImage:  metadataContainerImage,
 		ImagePullPolicy:         imagePullPolicy,
 		CPURequest:              resource.MustParse(cpuRequest),
 		CPULimit:                resource.MustParse(cpuLimit),
@@ -57,7 +59,7 @@ func LoadConfig(containerImage, imagePullPolicy, cpuRequest, cpuLimit, memoryReq
 }
 
 func FakeConfig() *Config {
-	return LoadConfig("fake-repo/fake-sidecar-image:v999.999.999-gke.0@sha256:c9cd4cde857ab8052f416609184e2900c0004838231ebf1c3817baa37f21d847", "Always", "250m", "250m", "256Mi", "256Mi", "5Gi", "5Gi")
+	return LoadConfig("fake-repo/fake-sidecar-image:v999.999.999-gke.0@sha256:c9cd4cde857ab8052f416609184e2900c0004838231ebf1c3817baa37f21d847", "Always", "250m", "250m", "256Mi", "256Mi", "5Gi", "5Gi", "fake-repo/fake-sidecar-image:v888.888.888-gke.0@sha256:c9cd4cde857ab8052f416609184e2900c0004838231ebf1c3817baa37f21d847")
 }
 
 func prepareResourceList(c *Config) (corev1.ResourceList, corev1.ResourceList) {
@@ -112,8 +114,9 @@ func populateResource(requestQuantity, limitQuantity *resource.Quantity, default
 // remaining values that are not specified by user are kept as the default config values.
 func (si *SidecarInjector) prepareConfig(annotations map[string]string) (*Config, error) {
 	config := &Config{
-		ContainerImage:  si.Config.ContainerImage,
-		ImagePullPolicy: si.Config.ImagePullPolicy,
+		ContainerImage:         si.Config.ContainerImage,
+		MetadataContainerImage: si.Config.MetadataContainerImage,
+		ImagePullPolicy:        si.Config.ImagePullPolicy,
 	}
 
 	jsonData, err := json.Marshal(annotations)

pkg/webhook/injection.go

@@ -71,6 +71,43 @@ func injectSidecarContainer(pod *corev1.Pod, config *Config, supportsNativeSidec
 	}
 }
 
+func injectMetadataPrefetchSidecarContainer(pod *corev1.Pod, config *Config, supportsNativeSidecar bool) {
+	// TODO(hime): Simplify this logic, this is very repetitive at first glance.
+	if supportsNativeSidecar {
+		spec := GetNativeMetadataPrefetchSidecarContainerSpec(pod, config)
+		index := getInjectIndexAfterContainer(pod.Spec.InitContainers, SidecarContainerName)
+		if len(spec.VolumeMounts) == 0 {
+			klog.Info("no volumes are requesting metadata prefetch, skipping metadata prefetch sidecar injection...")
+
+			return
+		}
+
+		if index == 0 {
+			klog.Warning("gke-gcsfuse-sidecar not found when attempting to inject metadata prefetch sidecar... skipping injection")
+
+			return
+		}
+
+		pod.Spec.InitContainers = insert(pod.Spec.InitContainers, spec, index)
+	} else {
+		spec := GetMetadataPrefetchSidecarContainerSpec(pod, config)
+		index := getInjectIndexAfterContainer(pod.Spec.Containers, SidecarContainerName)
+		if len(spec.VolumeMounts) == 0 {
+			klog.Info("no volumes are requesting metadata prefetch, skipping metadata prefetch sidecar injection...")
+
+			return
+		}
+
+		if index == 0 {
+			klog.Warning("gke-gcsfuse-sidecar not found when attempting to inject metadata prefetch sidecar... skipping injection")
+
+			return
+		}
+		// Inject sidecar.
+		pod.Spec.Containers = insert(pod.Spec.Containers, spec, index)
+	}
+}
+
 func insert(a []corev1.Container, value corev1.Container, index int) []corev1.Container {
 	// For index == len(a)
 	if len(a) == index {
@@ -85,7 +122,11 @@ func insert(a []corev1.Container, value corev1.Container, index int) []corev1.Co
 }
 
 func getInjectIndex(containers []corev1.Container) int {
-	idx, present := containerPresent(containers, IstioSidecarName)
+	return getInjectIndexAfterContainer(containers, IstioSidecarName)
+}
+
+func getInjectIndexAfterContainer(containers []corev1.Container, containerName string) int {
+	idx, present := containerPresent(containers, containerName)
 	if present {
 		return idx + 1
 	}

pkg/webhook/mutatingwebhook.go

@@ -111,6 +111,9 @@ func (si *SidecarInjector) Handle(_ context.Context, req admission.Request) admi
 	// Log pod mutation.
 	LogPodMutation(pod, config)
 
+	// Inject metadata prefetch sidecar.
+	injectMetadataPrefetchSidecarContainer(pod, config, supportsNativeSidecar)
+
 	marshaledPod, err := json.Marshal(pod)
 	if err != nil {
 		return admission.Errored(http.StatusBadRequest, fmt.Errorf("failed to marshal pod: %w", err))