From c418cf01641556dcaec28dafe61196456110a886 Mon Sep 17 00:00:00 2001 From: hime Date: Wed, 9 Oct 2024 04:43:46 +0000 Subject: [PATCH] Add metadata prefetch support. --- Makefile | 24 +++++++++- cmd/metadata_prefetch/Dockerfile | 44 +++++++++++++++++ cmd/metadata_prefetch/main.go | 73 +++++++++++++++++++++++++++++ cmd/webhook/main.go | 3 +- deploy/base/node/node.yaml | 1 + deploy/base/webhook/deployment.yaml | 6 +++ pkg/webhook/config.go | 15 +++--- pkg/webhook/injection.go | 43 ++++++++++++++++- pkg/webhook/mutatingwebhook.go | 3 ++ pkg/webhook/mutatingwebhook_test.go | 8 ++++ pkg/webhook/parsers.go | 11 +++++ pkg/webhook/sidecar_spec.go | 65 +++++++++++++++++++++++++ 12 files changed, 287 insertions(+), 9 deletions(-) create mode 100644 cmd/metadata_prefetch/Dockerfile create mode 100644 cmd/metadata_prefetch/main.go diff --git a/Makefile b/Makefile index cdb559fd5..e606a5466 100755 --- a/Makefile +++ b/Makefile @@ -29,10 +29,12 @@ IDENTITY_PROVIDER ?= $(shell kubectl get --raw /.well-known/openid-configuration DRIVER_BINARY = gcs-fuse-csi-driver SIDECAR_BINARY = gcs-fuse-csi-driver-sidecar-mounter WEBHOOK_BINARY = gcs-fuse-csi-driver-webhook +PREFETCH_BINARY = gcs-fuse-csi-driver-metadata-prefetch DRIVER_IMAGE = ${REGISTRY}/${DRIVER_BINARY} SIDECAR_IMAGE = ${REGISTRY}/${SIDECAR_BINARY} WEBHOOK_IMAGE = ${REGISTRY}/${WEBHOOK_BINARY} +PREFETCH_IMAGE = ${REGISTRY}/${PREFETCH_BINARY} DOCKER_BUILDX_ARGS ?= --push --builder multiarch-multiplatform-builder --build-arg STAGINGVERSION=${STAGINGVERSION} ifneq ("$(shell docker buildx build --help | grep 'provenance')", "") @@ -46,7 +48,7 @@ $(info DRIVER_IMAGE is ${DRIVER_IMAGE}) $(info SIDECAR_IMAGE is ${SIDECAR_IMAGE}) $(info WEBHOOK_IMAGE is ${WEBHOOK_IMAGE}) -all: driver sidecar-mounter webhook +all: driver sidecar-mounter webhook metadata-prefetch driver: mkdir -p ${BINDIR} @@ -56,6 +58,10 @@ sidecar-mounter: mkdir -p ${BINDIR} CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -mod vendor -ldflags "${LDFLAGS}" -o ${BINDIR}/${SIDECAR_BINARY} cmd/sidecar_mounter/main.go +metadata-prefetch: + mkdir -p ${BINDIR} + CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -mod vendor -ldflags "${LDFLAGS}" -o ${BINDIR}/${PREFETCH_BINARY} cmd/metadata_prefetch/main.go + webhook: mkdir -p ${BINDIR} CGO_ENABLED=0 GOOS=linux GOARCH=$(shell dpkg --print-architecture) go build -mod vendor -ldflags "${LDFLAGS}" -o ${BINDIR}/${WEBHOOK_BINARY} cmd/webhook/main.go @@ -129,18 +135,27 @@ ifeq (${BUILD_ARM}, true) make build-image-linux-arm64 docker manifest create ${DRIVER_IMAGE}:${STAGINGVERSION} ${DRIVER_IMAGE}:${STAGINGVERSION}_linux_amd64 ${DRIVER_IMAGE}:${STAGINGVERSION}_linux_arm64 docker manifest create ${SIDECAR_IMAGE}:${STAGINGVERSION} ${SIDECAR_IMAGE}:${STAGINGVERSION}_linux_amd64 ${SIDECAR_IMAGE}:${STAGINGVERSION}_linux_arm64 + docker manifest create ${PREFETCH_IMAGE}:${STAGINGVERSION} ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_amd64 ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_arm64 else docker manifest create ${DRIVER_IMAGE}:${STAGINGVERSION} ${DRIVER_IMAGE}:${STAGINGVERSION}_linux_amd64 docker manifest create ${SIDECAR_IMAGE}:${STAGINGVERSION} ${SIDECAR_IMAGE}:${STAGINGVERSION}_linux_amd64 + docker manifest create ${PREFETCH_IMAGE}:${STAGINGVERSION} ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_amd64 endif docker manifest create ${WEBHOOK_IMAGE}:${STAGINGVERSION} ${WEBHOOK_IMAGE}:${STAGINGVERSION}_linux_amd64 docker manifest push --purge ${DRIVER_IMAGE}:${STAGINGVERSION} docker manifest push --purge ${SIDECAR_IMAGE}:${STAGINGVERSION} + docker manifest push --purge ${PREFETCH_IMAGE}:${STAGINGVERSION} docker manifest push --purge ${WEBHOOK_IMAGE}:${STAGINGVERSION} build-image-linux-amd64: + docker buildx build ${DOCKER_BUILDX_ARGS} \ + --file ./cmd/metadata_prefetch/Dockerfile \ + --tag ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_amd64 \ + --platform linux/amd64 \ + --build-arg TARGETPLATFORM=linux/amd64 . + docker buildx build \ --file ./cmd/csi_driver/Dockerfile \ --tag validation_linux_amd64 \ @@ -164,6 +179,12 @@ build-image-linux-amd64: --platform linux/amd64 . build-image-linux-arm64: + docker buildx build ${DOCKER_BUILDX_ARGS} \ + --file ./cmd/sidecar_metadata_cache_populator/Dockerfile \ + --tag ${PREFETCH_IMAGE}:${STAGINGVERSION}_linux_arm64 \ + --platform linux/arm64 \ + --build-arg TARGETPLATFORM=linux/arm64 . + docker buildx build \ --file ./cmd/csi_driver/Dockerfile \ --tag validation_linux_arm64 \ @@ -198,6 +219,7 @@ generate-spec-yaml: cd ./deploy/overlays/${OVERLAY}; ${BINDIR}/kustomize edit set image gke.gcr.io/gcs-fuse-csi-driver=${DRIVER_IMAGE}:${STAGINGVERSION}; cd ./deploy/overlays/${OVERLAY}; ${BINDIR}/kustomize edit set image gke.gcr.io/gcs-fuse-csi-driver-webhook=${WEBHOOK_IMAGE}:${STAGINGVERSION}; cd ./deploy/overlays/${OVERLAY}; ${BINDIR}/kustomize edit add configmap gcsfusecsi-image-config --behavior=merge --disableNameSuffixHash --from-literal=sidecar-image=${SIDECAR_IMAGE}:${STAGINGVERSION}; + cd ./deploy/overlays/${OVERLAY}; ${BINDIR}/kustomize edit add configmap gcsfusecsi-image-config --behavior=merge --disableNameSuffixHash --from-literal=metadata-sidecar-image=${PREFETCH_IMAGE}:${STAGINGVERSION}; echo "[{\"op\": \"replace\",\"path\": \"/spec/tokenRequests/0/audience\",\"value\": \"${PROJECT}.svc.id.goog\"}]" > ./deploy/overlays/${OVERLAY}/project_patch_csi_driver.json echo "[{\"op\": \"replace\",\"path\": \"/webhooks/0/clientConfig/caBundle\",\"value\": \"${CA_BUNDLE}\"}]" > ./deploy/overlays/${OVERLAY}/caBundle_patch_MutatingWebhookConfiguration.json echo "[{\"op\": \"replace\",\"path\": \"/spec/template/spec/containers/0/env/1/value\",\"value\": \"${IDENTITY_PROVIDER}\"}]" > ./deploy/overlays/${OVERLAY}/identity_provider_patch_csi_node.json diff --git a/cmd/metadata_prefetch/Dockerfile b/cmd/metadata_prefetch/Dockerfile new file mode 100644 index 000000000..b94ff4362 --- /dev/null +++ b/cmd/metadata_prefetch/Dockerfile @@ -0,0 +1,44 @@ +# Copyright 2018 The Kubernetes Authors. +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build sidecar-mounter go binary +FROM golang:1.22.7 AS metadata-prefetch-builder + +ARG STAGINGVERSION + +WORKDIR /gcs-fuse-csi-driver +ADD . . +RUN make metadata-prefetch BINDIR=/bin + +# go/gke-releasing-policies#base-images +FROM gke.gcr.io/debian-base:bookworm-v1.0.4-gke.2 AS debian + +# go/gke-releasing-policies#base-images +FROM gcr.io/distroless/base-debian12 +ARG TARGETPLATFORM + +# Copy existing binaries. +COPY --from=debian /bin/ls /bin/ls + +# Copy dependencies. +COPY --from=debian /lib/x86_64-linux-gnu/libselinux.so.1 /lib/x86_64-linux-gnu/libselinux.so.1 +COPY --from=debian /lib/x86_64-linux-gnu/libc.so.6 /lib/x86_64-linux-gnu/libc.so.6 +COPY --from=debian /lib/x86_64-linux-gnu/libpcre2-8.so.0 /lib/x86_64-linux-gnu/libpcre2-8.so.0 +COPY --from=debian /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2 + +# Copy the built binaries +COPY --from=metadata-prefetch-builder /bin/gcs-fuse-csi-driver-metadata-prefetch /gcs-fuse-csi-driver-metadata-prefetch + +ENTRYPOINT ["/gcs-fuse-csi-driver-metadata-prefetch"] \ No newline at end of file diff --git a/cmd/metadata_prefetch/main.go b/cmd/metadata_prefetch/main.go new file mode 100644 index 000000000..ba6fde1ca --- /dev/null +++ b/cmd/metadata_prefetch/main.go @@ -0,0 +1,73 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "os" + "os/exec" + "os/signal" + "syscall" + + "k8s.io/klog/v2" +) + +func main() { + klog.InitFlags(nil) + flag.Parse() + + // Create cancellable context to pass into exec. + ctx, cancel := context.WithCancel(context.Background()) + + // Handle SIGTERM signal. + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGTERM) + + go func() { + <-sigs + klog.Info("Caught SIGTERM signal: Terminating...") + cancel() + + os.Exit(0) // Exit gracefully + }() + + // Start the "ls" command in the background. + // All our volumes are mounted under the /volumes/ directory. + cmd := exec.CommandContext(ctx, "ls", "-R", "/volumes/") + cmd.Stdout = nil // Connects file descriptor to the null device (os.DevNull). + + // TODO(hime): We should research stratergies to parallelize ls execution and speed up cache population. + err := cmd.Start() + if err == nil { + klog.Info("Running ls on bucket(s)") + err = cmd.Wait() + if err != nil { + klog.Errorf("Error while executing ls command: %v", err) + } else { + klog.Info("Metadata prefetch complete.") + } + } else { + klog.Errorf("Error starting ls command: %v.", err) + } + + klog.Info("Going to sleep...") + + // Keep the process running. + select {} +} diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 795d86048..6f6820d37 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -52,6 +52,7 @@ var ( ephemeralStorageRequest = flag.String("sidecar-ephemeral-storage-request", "5Gi", "The default ephemeral storage request for gcsfuse sidecar container.") ephemeralStorageLimit = flag.String("sidecar-ephemeral-storage-limit", "5Gi", "The default ephemeral storage limit for gcsfuse sidecar container.") sidecarImage = flag.String("sidecar-image", "", "The gcsfuse sidecar container image.") + metadataSidecarImage = flag.String("metadata-sidecar-image", "", "The gcsfuse sidecar container image.") // These are set at compile time. webhookVersion = "unknown" @@ -72,7 +73,7 @@ func main() { klog.Infof("Running Google Cloud Storage FUSE CSI driver admission webhook version %v, sidecar container image %v", webhookVersion, *sidecarImage) // Load webhook config - c := wh.LoadConfig(*sidecarImage, *imagePullPolicy, *cpuRequest, *cpuLimit, *memoryRequest, *memoryLimit, *ephemeralStorageRequest, *ephemeralStorageLimit) + c := wh.LoadConfig(*sidecarImage, *imagePullPolicy, *cpuRequest, *cpuLimit, *memoryRequest, *memoryLimit, *ephemeralStorageRequest, *ephemeralStorageLimit, *metadataSidecarImage) // Load config for manager, informers, listers kubeConfig := config.GetConfigOrDie() diff --git a/deploy/base/node/node.yaml b/deploy/base/node/node.yaml index 053239682..a319e9a5d 100755 --- a/deploy/base/node/node.yaml +++ b/deploy/base/node/node.yaml @@ -133,3 +133,4 @@ metadata: name: gcsfusecsi-image-config data: sidecar-image: gke.gcr.io/gcs-fuse-csi-driver-sidecar-mounter + metadata-sidecar-image: gke.gcr.io/gcs-fuse-csi-driver-metadata-prefetch diff --git a/deploy/base/webhook/deployment.yaml b/deploy/base/webhook/deployment.yaml index 2841190a0..e12481699 100644 --- a/deploy/base/webhook/deployment.yaml +++ b/deploy/base/webhook/deployment.yaml @@ -54,6 +54,7 @@ spec: - --sidecar-ephemeral-storage-limit=0 - --sidecar-ephemeral-storage-request=5Gi - --sidecar-image=$(SIDECAR_IMAGE) + - --metadata-sidecar-image=$(METADATA_SIDECAR_IMAGE) - --sidecar-image-pull-policy=$(SIDECAR_IMAGE_PULL_POLICY) - --cert-dir=/etc/tls-certs - --port=22030 @@ -66,6 +67,11 @@ spec: configMapKeyRef: name: gcsfusecsi-image-config key: sidecar-image + - name: METADATA_SIDECAR_IMAGE + valueFrom: + configMapKeyRef: + name: gcsfusecsi-image-config + key: metadata-sidecar-image resources: limits: cpu: 200m diff --git a/pkg/webhook/config.go b/pkg/webhook/config.go index ff2f3385c..9cb458eeb 100644 --- a/pkg/webhook/config.go +++ b/pkg/webhook/config.go @@ -27,8 +27,9 @@ import ( ) type Config struct { - ContainerImage string `json:"-"` - ImagePullPolicy string `json:"-"` + ContainerImage string `json:"-"` + MetadataContainerImage string `json:"-"` + ImagePullPolicy string `json:"-"` //nolint:tagliatelle CPURequest resource.Quantity `json:"gke-gcsfuse/cpu-request,omitempty"` //nolint:tagliatelle @@ -43,9 +44,10 @@ type Config struct { EphemeralStorageLimit resource.Quantity `json:"gke-gcsfuse/ephemeral-storage-limit,omitempty"` } -func LoadConfig(containerImage, imagePullPolicy, cpuRequest, cpuLimit, memoryRequest, memoryLimit, ephemeralStorageRequest, ephemeralStorageLimit string) *Config { +func LoadConfig(containerImage, imagePullPolicy, cpuRequest, cpuLimit, memoryRequest, memoryLimit, ephemeralStorageRequest, ephemeralStorageLimit, metadataContainerImage string) *Config { return &Config{ ContainerImage: containerImage, + MetadataContainerImage: metadataContainerImage, ImagePullPolicy: imagePullPolicy, CPURequest: resource.MustParse(cpuRequest), CPULimit: resource.MustParse(cpuLimit), @@ -57,7 +59,7 @@ func LoadConfig(containerImage, imagePullPolicy, cpuRequest, cpuLimit, memoryReq } func FakeConfig() *Config { - return LoadConfig("fake-repo/fake-sidecar-image:v999.999.999-gke.0@sha256:c9cd4cde857ab8052f416609184e2900c0004838231ebf1c3817baa37f21d847", "Always", "250m", "250m", "256Mi", "256Mi", "5Gi", "5Gi") + return LoadConfig("fake-repo/fake-sidecar-image:v999.999.999-gke.0@sha256:c9cd4cde857ab8052f416609184e2900c0004838231ebf1c3817baa37f21d847", "Always", "250m", "250m", "256Mi", "256Mi", "5Gi", "5Gi", "fake-repo/fake-sidecar-image:v888.888.888-gke.0@sha256:c9cd4cde857ab8052f416609184e2900c0004838231ebf1c3817baa37f21d847") } func prepareResourceList(c *Config) (corev1.ResourceList, corev1.ResourceList) { @@ -112,8 +114,9 @@ func populateResource(requestQuantity, limitQuantity *resource.Quantity, default // remaining values that are not specified by user are kept as the default config values. func (si *SidecarInjector) prepareConfig(annotations map[string]string) (*Config, error) { config := &Config{ - ContainerImage: si.Config.ContainerImage, - ImagePullPolicy: si.Config.ImagePullPolicy, + ContainerImage: si.Config.ContainerImage, + MetadataContainerImage: si.Config.MetadataContainerImage, + ImagePullPolicy: si.Config.ImagePullPolicy, } jsonData, err := json.Marshal(annotations) diff --git a/pkg/webhook/injection.go b/pkg/webhook/injection.go index 70a2636e1..2d7bee1de 100644 --- a/pkg/webhook/injection.go +++ b/pkg/webhook/injection.go @@ -71,6 +71,43 @@ func injectSidecarContainer(pod *corev1.Pod, config *Config, supportsNativeSidec } } +func injectMetadataPrefetchSidecarContainer(pod *corev1.Pod, config *Config, supportsNativeSidecar bool) { + // TODO(hime): Simplify this logic, this is very repetitive at first glance. + if supportsNativeSidecar { + spec := GetNativeMetadataPrefetchSidecarContainerSpec(pod, config) + index := getInjectIndexAfterContainer(pod.Spec.InitContainers, SidecarContainerName) + if len(spec.VolumeMounts) == 0 { + klog.Info("no volumes are requesting metadata prefetch, skipping metadata prefetch sidecar injection...") + + return + } + + if index == 0 { + klog.Warning("gke-gcsfuse-sidecar not found when attempting to inject metadata prefetch sidecar... skipping injection") + + return + } + + pod.Spec.InitContainers = insert(pod.Spec.InitContainers, spec, index) + } else { + spec := GetMetadataPrefetchSidecarContainerSpec(pod, config) + index := getInjectIndexAfterContainer(pod.Spec.Containers, SidecarContainerName) + if len(spec.VolumeMounts) == 0 { + klog.Info("no volumes are requesting metadata prefetch, skipping metadata prefetch sidecar injection...") + + return + } + + if index == 0 { + klog.Warning("gke-gcsfuse-sidecar not found when attempting to inject metadata prefetch sidecar... skipping injection") + + return + } + // Inject sidecar. + pod.Spec.Containers = insert(pod.Spec.Containers, spec, index) + } +} + func insert(a []corev1.Container, value corev1.Container, index int) []corev1.Container { // For index == len(a) if len(a) == index { @@ -85,7 +122,11 @@ func insert(a []corev1.Container, value corev1.Container, index int) []corev1.Co } func getInjectIndex(containers []corev1.Container) int { - idx, present := containerPresent(containers, IstioSidecarName) + return getInjectIndexAfterContainer(containers, IstioSidecarName) +} + +func getInjectIndexAfterContainer(containers []corev1.Container, containerName string) int { + idx, present := containerPresent(containers, containerName) if present { return idx + 1 } diff --git a/pkg/webhook/mutatingwebhook.go b/pkg/webhook/mutatingwebhook.go index b003a7107..92455f58e 100644 --- a/pkg/webhook/mutatingwebhook.go +++ b/pkg/webhook/mutatingwebhook.go @@ -111,6 +111,9 @@ func (si *SidecarInjector) Handle(_ context.Context, req admission.Request) admi // Log pod mutation. LogPodMutation(pod, config) + // Inject metadata prefetch sidecar. + injectMetadataPrefetchSidecarContainer(pod, config, supportsNativeSidecar) + marshaledPod, err := json.Marshal(pod) if err != nil { return admission.Errored(http.StatusBadRequest, fmt.Errorf("failed to marshal pod: %w", err)) diff --git a/pkg/webhook/mutatingwebhook_test.go b/pkg/webhook/mutatingwebhook_test.go index 410948f5b..9a27052d0 100644 --- a/pkg/webhook/mutatingwebhook_test.go +++ b/pkg/webhook/mutatingwebhook_test.go @@ -59,6 +59,7 @@ func TestPrepareConfig(t *testing.T) { }, wantConfig: &Config{ ContainerImage: FakeConfig().ContainerImage, + MetadataContainerImage: FakeConfig().MetadataContainerImage, ImagePullPolicy: FakeConfig().ImagePullPolicy, CPULimit: FakeConfig().CPULimit, CPURequest: FakeConfig().CPURequest, @@ -79,6 +80,7 @@ func TestPrepareConfig(t *testing.T) { }, wantConfig: &Config{ ContainerImage: FakeConfig().ContainerImage, + MetadataContainerImage: FakeConfig().MetadataContainerImage, ImagePullPolicy: FakeConfig().ImagePullPolicy, CPULimit: resource.MustParse("500m"), CPURequest: resource.MustParse("500m"), @@ -99,6 +101,7 @@ func TestPrepareConfig(t *testing.T) { }, wantConfig: &Config{ ContainerImage: FakeConfig().ContainerImage, + MetadataContainerImage: FakeConfig().MetadataContainerImage, ImagePullPolicy: FakeConfig().ImagePullPolicy, CPULimit: resource.MustParse("500m"), CPURequest: resource.MustParse("500m"), @@ -119,6 +122,7 @@ func TestPrepareConfig(t *testing.T) { }, wantConfig: &Config{ ContainerImage: FakeConfig().ContainerImage, + MetadataContainerImage: FakeConfig().MetadataContainerImage, ImagePullPolicy: FakeConfig().ImagePullPolicy, CPULimit: resource.Quantity{}, CPURequest: FakeConfig().CPURequest, @@ -139,6 +143,7 @@ func TestPrepareConfig(t *testing.T) { }, wantConfig: &Config{ ContainerImage: FakeConfig().ContainerImage, + MetadataContainerImage: FakeConfig().MetadataContainerImage, ImagePullPolicy: FakeConfig().ImagePullPolicy, CPULimit: resource.Quantity{}, CPURequest: resource.Quantity{}, @@ -162,6 +167,7 @@ func TestPrepareConfig(t *testing.T) { }, wantConfig: &Config{ ContainerImage: FakeConfig().ContainerImage, + MetadataContainerImage: FakeConfig().MetadataContainerImage, ImagePullPolicy: FakeConfig().ImagePullPolicy, CPULimit: resource.MustParse("500m"), CPURequest: resource.MustParse("100m"), @@ -185,6 +191,7 @@ func TestPrepareConfig(t *testing.T) { }, wantConfig: &Config{ ContainerImage: FakeConfig().ContainerImage, + MetadataContainerImage: FakeConfig().MetadataContainerImage, ImagePullPolicy: FakeConfig().ImagePullPolicy, CPULimit: resource.Quantity{}, CPURequest: resource.MustParse("100m"), @@ -208,6 +215,7 @@ func TestPrepareConfig(t *testing.T) { }, wantConfig: &Config{ ContainerImage: FakeConfig().ContainerImage, + MetadataContainerImage: FakeConfig().MetadataContainerImage, ImagePullPolicy: FakeConfig().ImagePullPolicy, CPULimit: resource.MustParse("500m"), CPURequest: resource.Quantity{}, diff --git a/pkg/webhook/parsers.go b/pkg/webhook/parsers.go index 45aa1d1aa..b3cf06ab0 100644 --- a/pkg/webhook/parsers.go +++ b/pkg/webhook/parsers.go @@ -27,6 +27,17 @@ import ( var minimumSupportedVersion = version.MustParseGeneric("1.29.0") +func ParseBool(str string) (bool, error) { + switch str { + case "True", "true": + return true, nil + case "False", "false": + return false, nil + default: + return false, fmt.Errorf("could not parse string to bool: the acceptable values for %q are 'True', 'true', 'false' or 'False'", str) + } +} + // parseSidecarContainerImage supports our Privately Hosted Sidecar Image option // by iterating the container list and finding a container named "gke-gcsfuse-sidecar" // If we find "gke-gcsfuse-sidecar": diff --git a/pkg/webhook/sidecar_spec.go b/pkg/webhook/sidecar_spec.go index 2ad2af741..adf067ba0 100644 --- a/pkg/webhook/sidecar_spec.go +++ b/pkg/webhook/sidecar_spec.go @@ -18,6 +18,8 @@ limitations under the License. package webhook import ( + "path/filepath" + corev1 "k8s.io/api/core/v1" "k8s.io/klog/v2" "k8s.io/utils/ptr" @@ -25,6 +27,7 @@ import ( const ( SidecarContainerName = "gke-gcsfuse-sidecar" + SidecarMetadataPrefetchName = "gke-gcsfuse-metadata-prefetch" SidecarContainerTmpVolumeName = "gke-gcsfuse-tmp" SidecarContainerTmpVolumeMountPath = "/gcsfuse-tmp" SidecarContainerBufferVolumeName = "gke-gcsfuse-buffer" @@ -118,6 +121,68 @@ func GetSidecarContainerSpec(c *Config) corev1.Container { return container } +func GetNativeMetadataPrefetchSidecarContainerSpec(pod *corev1.Pod, c *Config) corev1.Container { + container := GetMetadataPrefetchSidecarContainerSpec(pod, c) + container.Env = append(container.Env, corev1.EnvVar{Name: "NATIVE_SIDECAR", Value: "TRUE"}) + container.RestartPolicy = ptr.To(corev1.ContainerRestartPolicyAlways) + + return container +} + +func GetMetadataPrefetchSidecarContainerSpec(pod *corev1.Pod, c *Config) corev1.Container { + limits, requests := prepareResourceList(c) + + // The sidecar container follows Restricted Pod Security Standard, + // see https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + container := corev1.Container{ + Name: SidecarMetadataPrefetchName, + Image: c.MetadataContainerImage, + ImagePullPolicy: corev1.PullPolicy(c.ImagePullPolicy), + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + ReadOnlyRootFilesystem: ptr.To(true), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + corev1.Capability("ALL"), + }, + }, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To(int64(NobodyUID)), + RunAsGroup: ptr.To(int64(NobodyGID)), + }, + Args: []string{ + "--v=5", + }, + Resources: corev1.ResourceRequirements{ + // We should change these resources. + Limits: limits, + Requests: requests, + }, + VolumeMounts: []corev1.VolumeMount{}, + } + + for _, v := range pod.Spec.Volumes { + if v.CSI == nil { + // We don't log because it can generate lots of trash. + continue + } + if v.CSI.Driver == "gcsfuse.csi.storage.gke.io" { + enableMetaPrefetch, err := ParseBool(v.CSI.VolumeAttributes["gcsfuseMetadataPrefetchOnMount"]) + if err != nil { + klog.Errorf("failed to parse bool %v", enableMetaPrefetch) + + continue + } + if enableMetaPrefetch { + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: v.Name, MountPath: filepath.Join("/volumes/", v.Name), ReadOnly: true}) + } + } + } + + return container +} + // GetSidecarContainerVolumeSpec returns volumes required by the sidecar container, // skipping the existing custom volumes. func GetSidecarContainerVolumeSpec(existingVolumes ...corev1.Volume) []corev1.Volume {