diff --git a/pkg/sidecar_mounter/sidecar_mounter.go b/pkg/sidecar_mounter/sidecar_mounter.go index 0b676bcf6..b4cc2f529 100644 --- a/pkg/sidecar_mounter/sidecar_mounter.go +++ b/pkg/sidecar_mounter/sidecar_mounter.go @@ -250,6 +250,8 @@ func (mc *MountConfig) prepareMountArgs() (map[string]string, map[string]string) value = GCSFuseAppName + "-" + value case flag == util.DisableFileCacheKey: configFileFlagMap["cache-dir"] = "" + klog.Info("gcsfuse file cache is disabled.") + continue } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 67be773a5..c00ff9397 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -105,6 +105,7 @@ var _ = ginkgo.Describe("E2E Test Suite", func() { testsuites.InitGcsFuseCSIPerformanceTestSuite, testsuites.InitGcsFuseCSISubPathTestSuite, testsuites.InitGcsFuseCSIAutoTerminationTestSuite, + testsuites.InitGcsFuseCSIFileCacheTestSuite, } testDriver := InitGCSFuseCSITestDriver(c, m, *bucketLocation, *skipGcpSaTest) diff --git a/test/e2e/specs/specs.go b/test/e2e/specs/specs.go index a97d4c9ff..24a7236ad 100644 --- a/test/e2e/specs/specs.go +++ b/test/e2e/specs/specs.go @@ -60,6 +60,7 @@ const ( ForceNewBucketPrefix = "gcsfuse-csi-force-new-bucket" SubfolderInBucketPrefix = "gcsfuse-csi-subfolder-in-bucket" MultipleBucketsPrefix = "gcsfuse-csi-multiple-buckets" + DisableFileCachePrefix = "gcsfuse-csi-disable-file-cache" ImplicitDirsPath = "implicit-dir" InvalidVolume = "" @@ -280,6 +281,14 @@ func (t *TestPod) setupVolume(volumeResource *storageframework.VolumeResource, n t.pod.Spec.Volumes = append(t.pod.Spec.Volumes, volume) } +func (t *TestPod) SetupCacheVolumeMount(mountPath string) { + volumeMount := v1.VolumeMount{ + Name: webhook.SidecarContainerCacheVolumeName, + MountPath: mountPath, + } + t.pod.Spec.Containers[0].VolumeMounts = append(t.pod.Spec.Containers[0].VolumeMounts, volumeMount) +} + func (t *TestPod) SetName(name string) { t.pod.Name = name } @@ -956,12 +965,11 @@ func CreateImplicitDirInBucket(dirPath, bucketName string) { } } -func CreateEmptyFileInBucket(fileName, bucketName string) { - f, err := os.Create(fileName) +func CreateTestFileInBucket(fileName, bucketName string) { + err := os.WriteFile(fileName, []byte(fileName), 0o644) if err != nil { - framework.Failf("Failed to create an empty data file: %v", err) + framework.Failf("Failed to create a test file: %v", err) } - f.Close() defer func() { err = os.Remove(fileName) if err != nil { @@ -971,6 +979,6 @@ func CreateEmptyFileInBucket(fileName, bucketName string) { //nolint:gosec if output, err := exec.Command("gsutil", "cp", fileName, fmt.Sprintf("gs://%v", bucketName)).CombinedOutput(); err != nil { - framework.Failf("Failed to create an empty file in GCS bucket: %v, output: %s", err, output) + framework.Failf("Failed to create a test file in GCS bucket: %v, output: %s", err, output) } } diff --git a/test/e2e/testdriver.go b/test/e2e/testdriver.go index b69fbb114..35eae48f2 100644 --- a/test/e2e/testdriver.go +++ b/test/e2e/testdriver.go @@ -52,6 +52,7 @@ type gcsVolume struct { bucketName string serviceAccountNamespace string mountOptions string + disableFileCache string shared bool readOnly bool } @@ -181,6 +182,7 @@ func (n *GCSFuseCSITestDriver) CreateVolume(ctx context.Context, config *storage } mountOptions := "debug_gcs,debug_fuse,debug_fs" + disableFileCache := "" switch config.Prefix { case specs.NonRootVolumePrefix: mountOptions += ",uid=1001" @@ -193,19 +195,22 @@ func (n *GCSFuseCSITestDriver) CreateVolume(ctx context.Context, config *storage dirPath := uuid.NewString() specs.CreateImplicitDirInBucket(dirPath, bucketName) mountOptions += ",only-dir=" + dirPath + case specs.DisableFileCachePrefix: + disableFileCache = "true" } v := &gcsVolume{ bucketName: bucketName, serviceAccountNamespace: config.Framework.Namespace.Name, mountOptions: mountOptions, + disableFileCache: disableFileCache, } if !isMultipleBucketsPrefix { n.volumeStore = append(n.volumeStore, v) } - if config.Prefix == "" { + if config.Prefix == "" || config.Prefix == specs.DisableFileCachePrefix { // Use config.Prefix to pass the bucket names back to the test suite. config.Prefix = bucketName } @@ -226,7 +231,10 @@ func (v *gcsVolume) DeleteVolume(_ context.Context) { func (n *GCSFuseCSITestDriver) GetPersistentVolumeSource(readOnly bool, _ string, volume storageframework.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) { gv, _ := volume.(*gcsVolume) - va := map[string]string{"mountOptions": gv.mountOptions} + va := map[string]string{ + driver.VolumeContextKeyMountOptions: gv.mountOptions, + driver.VolumeContextKeyDisableFileCache: gv.disableFileCache, + } return &v1.PersistentVolumeSource{ CSI: &v1.CSIPersistentVolumeSource{ @@ -243,8 +251,9 @@ func (n *GCSFuseCSITestDriver) GetVolume(config *storageframework.PerTestConfig, gv, _ := volume.(*gcsVolume) return map[string]string{ - "bucketName": gv.bucketName, - "mountOptions": gv.mountOptions, + driver.VolumeContextKeyBucketName: gv.bucketName, + driver.VolumeContextKeyMountOptions: gv.mountOptions, + driver.VolumeContextKeyDisableFileCache: gv.disableFileCache, }, gv.shared, gv.readOnly } diff --git a/test/e2e/testsuites/auto_termination.go b/test/e2e/testsuites/auto_termination.go index 187deef47..f18ab2886 100644 --- a/test/e2e/testsuites/auto_termination.go +++ b/test/e2e/testsuites/auto_termination.go @@ -95,7 +95,7 @@ func (t *gcsFuseCSIAutoTerminationTestSuite) DefineTests(driver storageframework tPod1 := specs.NewTestPod(f.ClientSet, f.Namespace) tPod1.SetRestartPolicy(v1.RestartPolicyAlways) tPod1.SetGracePeriod(600) - tPod1.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod1.SetupVolume(l.volumeResource, volumeName, mountPath, false) cmd := []string{ fmt.Sprintf("handle_term() { echo 'Caught SIGTERM signal!'; echo 'hello world' > %v/data && grep 'hello world' %v/data; sleep 5; exit 0; };", mountPath, mountPath), "trap handle_term SIGTERM;", @@ -118,7 +118,7 @@ func (t *gcsFuseCSIAutoTerminationTestSuite) DefineTests(driver storageframework ginkgo.By("Configuring the second pod") tPod2 := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod2.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod2.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the second pod") tPod2.Create(ctx) @@ -139,7 +139,7 @@ func (t *gcsFuseCSIAutoTerminationTestSuite) DefineTests(driver storageframework ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetRestartPolicy(v1.RestartPolicyNever) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand(fmt.Sprintf("echo 'hello world' > %v/data && grep 'hello world' %v/data", mountPath, mountPath)) ginkgo.By("Deploying the pod") @@ -157,7 +157,7 @@ func (t *gcsFuseCSIAutoTerminationTestSuite) DefineTests(driver storageframework ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetRestartPolicy(v1.RestartPolicyNever) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand("sleep 10; exit 1;") ginkgo.By("Deploying the pod") @@ -175,7 +175,7 @@ func (t *gcsFuseCSIAutoTerminationTestSuite) DefineTests(driver storageframework ginkgo.By("Configuring the first pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetRestartPolicy(v1.RestartPolicyAlways) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand(fmt.Sprintf("echo 'hello world' > %v/data && grep 'hello world' %v/data", mountPath, mountPath)) ginkgo.By("Deploying the first pod") @@ -197,7 +197,7 @@ func (t *gcsFuseCSIAutoTerminationTestSuite) DefineTests(driver storageframework ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetRestartPolicy(v1.RestartPolicyAlways) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand("sleep 10; exit 1;") ginkgo.By("Deploying the pod") @@ -219,7 +219,7 @@ func (t *gcsFuseCSIAutoTerminationTestSuite) DefineTests(driver storageframework ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetRestartPolicy(v1.RestartPolicyOnFailure) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) cmd := []string{ "sleep 10;", fmt.Sprintf("if [ -f %v/testfile ]; then echo 'Completed Successfully!'; exit 0; fi;", mountPath), @@ -244,7 +244,7 @@ func (t *gcsFuseCSIAutoTerminationTestSuite) DefineTests(driver storageframework ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetRestartPolicy(v1.RestartPolicyAlways) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand("sleep 10; exit 1;") ginkgo.By("Deploying the pod") diff --git a/test/e2e/testsuites/failed_mount.go b/test/e2e/testsuites/failed_mount.go index 6f637eeec..437618dc3 100644 --- a/test/e2e/testsuites/failed_mount.go +++ b/test/e2e/testsuites/failed_mount.go @@ -95,7 +95,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the pod") tPod.Create(ctx) @@ -116,7 +116,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the pod") tPod.Create(ctx) @@ -133,7 +133,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying a Kubernetes service account without access to the GCS bucket") saName := "sa-without-access" @@ -163,7 +163,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetAnnotations(map[string]string{ "gke-gcsfuse/volumes": "false", @@ -184,7 +184,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand("touch /mnt/test/test_file && while true; do echo $(date) >> /mnt/test/test_file; done") tPod.SetAnnotations(map[string]string{ @@ -206,7 +206,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the pod") tPod.Create(ctx) @@ -223,7 +223,7 @@ func (t *gcsFuseCSIFailedMountTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetAnnotations(map[string]string{ "gke-gcsfuse/memory-limit": "1000000000Gi", diff --git a/test/e2e/testsuites/file_cache.go b/test/e2e/testsuites/file_cache.go new file mode 100644 index 000000000..2e0bcdfdc --- /dev/null +++ b/test/e2e/testsuites/file_cache.go @@ -0,0 +1,196 @@ +/* +Copyright 2018 The Kubernetes Authors. +Copyright 2022 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + "fmt" + + "github.com/google/uuid" + "github.com/googlecloudplatform/gcs-fuse-csi-driver/pkg/webhook" + "github.com/googlecloudplatform/gcs-fuse-csi-driver/test/e2e/specs" + "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/kubernetes/test/e2e/framework" + e2evolume "k8s.io/kubernetes/test/e2e/framework/volume" + storageframework "k8s.io/kubernetes/test/e2e/storage/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +type gcsFuseCSIFileCacheTestSuite struct { + tsInfo storageframework.TestSuiteInfo +} + +// InitGcsFuseCSIFileCacheTestSuite returns gcsFuseCSIFileCacheTestSuite that implements TestSuite interface. +func InitGcsFuseCSIFileCacheTestSuite() storageframework.TestSuite { + return &gcsFuseCSIFileCacheTestSuite{ + tsInfo: storageframework.TestSuiteInfo{ + Name: "fileCache", + TestPatterns: []storageframework.TestPattern{ + // storageframework.DefaultFsCSIEphemeralVolume, + storageframework.DefaultFsPreprovisionedPV, + // storageframework.DefaultFsDynamicPV, + }, + }, + } +} + +func (t *gcsFuseCSIFileCacheTestSuite) GetTestSuiteInfo() storageframework.TestSuiteInfo { + return t.tsInfo +} + +func (t *gcsFuseCSIFileCacheTestSuite) SkipUnsupportedTests(_ storageframework.TestDriver, _ storageframework.TestPattern) { +} + +func (t *gcsFuseCSIFileCacheTestSuite) DefineTests(driver storageframework.TestDriver, pattern storageframework.TestPattern) { + type local struct { + config *storageframework.PerTestConfig + volumeResource *storageframework.VolumeResource + } + var l local + ctx := context.Background() + + // Beware that it also registers an AfterEach which renders f unusable. Any code using + // f must run inside an It or Context callback. + f := framework.NewFrameworkWithCustomTimeouts("file-cache", storageframework.GetDriverTimeouts(driver)) + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + init := func(configPrefix ...string) { + l = local{} + l.config = driver.PrepareTest(ctx, f) + if len(configPrefix) > 0 { + l.config.Prefix = configPrefix[0] + } + l.volumeResource = storageframework.CreateVolumeResource(ctx, driver, l.config, pattern, e2evolume.SizeRange{}) + } + + cleanup := func() { + var cleanUpErrs []error + cleanUpErrs = append(cleanUpErrs, l.volumeResource.CleanupResource(ctx)) + err := utilerrors.NewAggregate(cleanUpErrs) + framework.ExpectNoError(err, "while cleaning up") + } + + ginkgo.It("should cache the data", func() { + init() + defer cleanup() + + // The test driver uses config.Prefix to pass the bucket names back to the test suite. + bucketName := l.config.Prefix + + // Create files using gsutil + fileName := uuid.NewString() + specs.CreateTestFileInBucket(fileName, bucketName) + + ginkgo.By("Configuring the first pod") + tPod := specs.NewTestPod(f.ClientSet, f.Namespace) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) + // Mount the gcsfuse cache volume to the test container + tPod.SetupCacheVolumeMount("/cache") + + cacheSubfolder := volumeName + if l.volumeResource.Pv != nil { + cacheSubfolder = l.volumeResource.Pv.Name + } + + ginkgo.By("Deploying the first pod") + tPod.Create(ctx) + defer tPod.Cleanup(ctx) + + ginkgo.By("Checking that the first pod is running") + tPod.WaitForRunning(ctx) + + ginkgo.By("Checking that the first pod command exits with no error") + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mount | grep %v | grep rw,", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("cat %v/%v", mountPath, fileName)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("grep '%v' /cache/.volumes/%v/gcsfuse-file-cache/%v/%v", fileName, cacheSubfolder, bucketName, fileName)) + }) + + ginkgo.It("should cache the data using custom cache volume", func() { + init() + defer cleanup() + + // The test driver uses config.Prefix to pass the bucket names back to the test suite. + bucketName := l.config.Prefix + + // Create files using gsutil + fileName := uuid.NewString() + specs.CreateTestFileInBucket(fileName, bucketName) + + ginkgo.By("Configuring the pod") + tPod := specs.NewTestPod(f.ClientSet, f.Namespace) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) + tPVC := specs.NewTestPVC(f.ClientSet, f.Namespace, "custom-cache", "standard-rwo", "5Gi", v1.ReadWriteOnce) + tPod.SetupVolume(&storageframework.VolumeResource{Pvc: tPVC.PVC}, webhook.SidecarContainerCacheVolumeName, "", false) + tPod.SetupCacheVolumeMount("/cache") + tPod.SetNonRootSecurityContext(0, 0, 1000) + + cacheSubfolder := volumeName + if l.volumeResource.Pv != nil { + cacheSubfolder = l.volumeResource.Pv.Name + } + + ginkgo.By("Creating the PVC") + tPVC.Create(ctx) + defer tPVC.Cleanup(ctx) + + ginkgo.By("Deploying the pod") + tPod.Create(ctx) + defer tPod.Cleanup(ctx) + + ginkgo.By("Checking that the pod is running") + tPod.WaitForRunning(ctx) + + ginkgo.By("Checking that the pod command exits with no error") + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mount | grep %v | grep rw,", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("cat %v/%v", mountPath, fileName)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("grep '%v' /cache/.volumes/%v/gcsfuse-file-cache/%v/%v", fileName, cacheSubfolder, bucketName, fileName)) + }) + + ginkgo.It("should not cache the data when the file cache is disabled", func() { + init(specs.DisableFileCachePrefix) + defer cleanup() + + // The test driver uses config.Prefix to pass the bucket names back to the test suite. + bucketName := l.config.Prefix + + // Create files using gsutil + fileName := uuid.NewString() + specs.CreateTestFileInBucket(fileName, bucketName) + + ginkgo.By("Configuring the first pod") + tPod := specs.NewTestPod(f.ClientSet, f.Namespace) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) + // Mount the gcsfuse cache volume to the test container + tPod.SetupCacheVolumeMount("/cache") + + ginkgo.By("Deploying the first pod") + tPod.Create(ctx) + defer tPod.Cleanup(ctx) + + ginkgo.By("Checking that the first pod is running") + tPod.WaitForRunning(ctx) + + ginkgo.By("Checking that the first pod command exits with no error") + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("mount | grep %v | grep rw,", mountPath)) + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, fmt.Sprintf("cat %v/%v", mountPath, fileName)) + // the cache volume should be empty + tPod.VerifyExecInPodSucceed(f, specs.TesterContainerName, "[ ! -d '/cache/.volumes' ] && exit 0 || exit 1") + }) +} diff --git a/test/e2e/testsuites/gcsfuse_integration.go b/test/e2e/testsuites/gcsfuse_integration.go index 869243978..62d06197f 100644 --- a/test/e2e/testsuites/gcsfuse_integration.go +++ b/test/e2e/testsuites/gcsfuse_integration.go @@ -104,7 +104,7 @@ func (t *gcsFuseCSIGCSFuseIntegrationTestSuite) DefineTests(driver storageframew } l.volumeResource.VolSource.CSI.VolumeAttributes["mountOptions"] = mo - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, readOnly, mountOptions...) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, readOnly, mountOptions...) tPod.SetAnnotations(map[string]string{ "gke-gcsfuse/cpu-limit": "250m", "gke-gcsfuse/memory-limit": sidecarMemoryLimit, diff --git a/test/e2e/testsuites/multivolume.go b/test/e2e/testsuites/multivolume.go index b7965b347..977590caa 100644 --- a/test/e2e/testsuites/multivolume.go +++ b/test/e2e/testsuites/multivolume.go @@ -95,7 +95,7 @@ func (t *gcsFuseCSIMultiVolumeTestSuite) DefineTests(driver storageframework.Tes testTwoPodsSameBucket := func(diffVol, sameNode bool) { ginkgo.By("Configuring the first pod") tPod1 := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod1.SetupVolume(l.volumeResourceList[0], "test-gcsfuse-volume", mountPath, false) + tPod1.SetupVolume(l.volumeResourceList[0], volumeName, mountPath, false) ginkgo.By("Deploying the first pod") tPod1.Create(ctx) @@ -110,7 +110,7 @@ func (t *gcsFuseCSIMultiVolumeTestSuite) DefineTests(driver storageframework.Tes if diffVol { volIndex = 1 } - tPod2.SetupVolume(l.volumeResourceList[volIndex], "test-gcsfuse-volume", mountPath, false) + tPod2.SetupVolume(l.volumeResourceList[volIndex], volumeName, mountPath, false) tPod2.SetNodeAffinity(tPod1.GetNode(), sameNode) ginkgo.By("Deploying the second pod") @@ -137,7 +137,7 @@ func (t *gcsFuseCSIMultiVolumeTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) for i, vr := range l.volumeResourceList { - tPod.SetupVolume(vr, fmt.Sprintf("test-gcsfuse-volume-%v", i), fmt.Sprintf("%v/%v", mountPath, i), false) + tPod.SetupVolume(vr, fmt.Sprintf("%v-%v", volumeName, i), fmt.Sprintf("%v/%v", mountPath, i), false) } ginkgo.By("Deploying the pod") @@ -285,7 +285,7 @@ func (t *gcsFuseCSIMultiVolumeTestSuite) DefineTests(driver storageframework.Tes ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResourceList[0], "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResourceList[0], volumeName, mountPath, false) ginkgo.By("Sleeping 2 minutes for the service account being propagated") time.Sleep(time.Minute * 2) diff --git a/test/e2e/testsuites/performance.go b/test/e2e/testsuites/performance.go index 22a41f4f4..be30a64c4 100644 --- a/test/e2e/testsuites/performance.go +++ b/test/e2e/testsuites/performance.go @@ -197,7 +197,7 @@ func (t *gcsFuseCSIPerformanceTestSuite) DefineTests(driver storageframework.Tes tPod.SetImage(specs.UbuntuImage) tPod.SetResource("2", "5Gi", "5Gi") mountPath := "/gcs" - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false, "implicit-dirs", "max-conns-per-host=100", "client-protocol=http1") + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false, "implicit-dirs", "max-conns-per-host=100", "client-protocol=http1") tPod.SetAnnotations(map[string]string{ "gke-gcsfuse/cpu-limit": "10", "gke-gcsfuse/memory-limit": "2Gi", diff --git a/test/e2e/testsuites/subpath.go b/test/e2e/testsuites/subpath.go index ca8af9f6a..3d3e9c9fe 100644 --- a/test/e2e/testsuites/subpath.go +++ b/test/e2e/testsuites/subpath.go @@ -95,8 +95,8 @@ func (t *gcsFuseCSISubPathTestSuite) DefineTests(driver storageframework.TestDri tPod1.SetAnnotations(map[string]string{ "gke-gcsfuse/memory-limit": "100Mi", }) - tPod1.SetupVolumeWithSubPath(l.volumeResource, "test-gcsfuse-volume", mountPath+"1", false, "subpath1", false /* add the first volume */) - tPod1.SetupVolumeWithSubPath(nil, "test-gcsfuse-volume", mountPath+"2", false, "subpath2", true /* reuse the first volume */) + tPod1.SetupVolumeWithSubPath(l.volumeResource, volumeName, mountPath+"1", false, "subpath1", false /* add the first volume */) + tPod1.SetupVolumeWithSubPath(nil, volumeName, mountPath+"2", false, "subpath2", true /* reuse the first volume */) ginkgo.By("Deploying the first pod") tPod1.Create(ctx) @@ -118,7 +118,7 @@ func (t *gcsFuseCSISubPathTestSuite) DefineTests(driver storageframework.TestDri tPod2.SetAnnotations(map[string]string{ "gke-gcsfuse/memory-limit": "100Mi", }) - tPod2.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod2.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the second pod") tPod2.Create(ctx) @@ -149,8 +149,8 @@ func (t *gcsFuseCSISubPathTestSuite) DefineTests(driver storageframework.TestDri tPod.SetAnnotations(map[string]string{ "gke-gcsfuse/memory-limit": "100Mi", }) - tPod.SetupVolumeWithSubPath(l.volumeResource, "test-gcsfuse-volume", mountPath+"1", false, "subpath1", false /* add the first volume */) - tPod.SetupVolumeWithSubPath(nil, "test-gcsfuse-volume", mountPath+"2", false, "subpath2", true /* reuse the first volume */) + tPod.SetupVolumeWithSubPath(l.volumeResource, volumeName, mountPath+"1", false, "subpath1", false /* add the first volume */) + tPod.SetupVolumeWithSubPath(nil, volumeName, mountPath+"2", false, "subpath2", true /* reuse the first volume */) ginkgo.By("Deploying the pod") tPod.Create(ctx) @@ -176,13 +176,13 @@ func (t *gcsFuseCSISubPathTestSuite) DefineTests(driver storageframework.TestDri // Create files using gsutil file1 := uuid.NewString() file2 := uuid.NewString() - specs.CreateEmptyFileInBucket(file1, bucketName) - specs.CreateEmptyFileInBucket(file2, bucketName) + specs.CreateTestFileInBucket(file1, bucketName) + specs.CreateTestFileInBucket(file2, bucketName) ginkgo.By("Configuring the pod") tPod1 := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod1.SetupVolumeWithSubPath(l.volumeResource, "test-gcsfuse-volume", mountPath+"1", false, file1, false /* add the first volume */) - tPod1.SetupVolumeWithSubPath(nil, "test-gcsfuse-volume", mountPath+"2", false, file2, true /* reuse the first volume */) + tPod1.SetupVolumeWithSubPath(l.volumeResource, volumeName, mountPath+"1", false, file1, false /* add the first volume */) + tPod1.SetupVolumeWithSubPath(nil, volumeName, mountPath+"2", false, file2, true /* reuse the first volume */) ginkgo.By("Deploying the pod") tPod1.Create(ctx) @@ -205,7 +205,7 @@ func (t *gcsFuseCSISubPathTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the writer pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetName("gcsfuse-volume-tester-writer") - tPod.SetupVolumeWithSubPath(l.volumeResource, "test-gcsfuse-volume", mountPath, false, "subpath", false /* add the first volume */) + tPod.SetupVolumeWithSubPath(l.volumeResource, volumeName, mountPath, false, "subpath", false /* add the first volume */) ginkgo.By("Deploying the writer pod") tPod.Create(ctx) @@ -226,8 +226,8 @@ func (t *gcsFuseCSISubPathTestSuite) DefineTests(driver storageframework.TestDri if pattern.VolType == storageframework.CSIInlineVolume && l.volumeResource.VolSource != nil { l.volumeResource.VolSource.CSI.ReadOnly = ptr.To(true) } - tPod.SetupVolumeWithSubPath(l.volumeResource, "test-gcsfuse-volume", mountPath+"1", true, "subpath", false /* add the first volume */) - tPod.SetupVolumeWithSubPath(nil, "test-gcsfuse-volume", mountPath+"2", true, "subpath/data", true /* reuse the first volume */) + tPod.SetupVolumeWithSubPath(l.volumeResource, volumeName, mountPath+"1", true, "subpath", false /* add the first volume */) + tPod.SetupVolumeWithSubPath(nil, volumeName, mountPath+"2", true, "subpath/data", true /* reuse the first volume */) ginkgo.By("Deploying the reader pod") tPod.Create(ctx) diff --git a/test/e2e/testsuites/volumes.go b/test/e2e/testsuites/volumes.go index c07cee9fe..ff4d5527c 100644 --- a/test/e2e/testsuites/volumes.go +++ b/test/e2e/testsuites/volumes.go @@ -34,7 +34,10 @@ import ( "k8s.io/utils/ptr" ) -const mountPath = "/mnt/test" +const ( + mountPath = "/mnt/test" + volumeName = "test-gcsfuse-volume" +) type gcsFuseCSIVolumesTestSuite struct { tsInfo storageframework.TestSuiteInfo @@ -96,7 +99,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the first pod") tPod1 := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod1.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod1.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the first pod") tPod1.Create(ctx) @@ -113,7 +116,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the second pod") tPod2 := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod2.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod2.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the second pod") tPod2.Create(ctx) @@ -134,7 +137,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the writer pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetName("gcsfuse-volume-tester-writer") - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the writer pod") tPod.Create(ctx) @@ -155,7 +158,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri if pattern.VolType == storageframework.CSIInlineVolume && l.volumeResource.VolSource != nil { l.volumeResource.VolSource.CSI.ReadOnly = ptr.To(true) } - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, true) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, true) ginkgo.By("Deploying the reader pod") tPod.Create(ctx) @@ -179,7 +182,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the first pod") tPod1 := specs.NewTestPod(f.ClientSet, f.Namespace) tPod1.SetNonRootSecurityContext(1001, 2002, 0) - tPod1.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod1.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the first pod") tPod1.Create(ctx) @@ -196,7 +199,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the second pod") tPod2 := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod2.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod2.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the second pod") tPod2.Create(ctx) @@ -217,7 +220,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the first pod") tPod1 := specs.NewTestPod(f.ClientSet, f.Namespace) tPod1.SetNonRootSecurityContext(1001, 2002, 3003) - tPod1.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod1.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the first pod") tPod1.Create(ctx) @@ -234,7 +237,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the second pod") tPod2 := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod2.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod2.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the second pod") tPod2.Create(ctx) @@ -258,7 +261,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the pod") tPod.Create(ctx) @@ -279,7 +282,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetCustomSidecarContainerImage() - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the pod") tPod.Create(ctx) @@ -302,7 +305,7 @@ func (t *gcsFuseCSIVolumesTestSuite) DefineTests(driver storageframework.TestDri ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPVC := specs.NewTestPVC(f.ClientSet, f.Namespace, "custom-buffer", "standard-rwo", "5Gi", v1.ReadWriteOnce) tPod.SetupVolume(&storageframework.VolumeResource{Pvc: tPVC.PVC}, webhook.SidecarContainerBufferVolumeName, "", false) tPod.SetNonRootSecurityContext(0, 0, 1000) diff --git a/test/e2e/testsuites/workloads.go b/test/e2e/testsuites/workloads.go index 1e9ef795a..aa93bc408 100644 --- a/test/e2e/testsuites/workloads.go +++ b/test/e2e/testsuites/workloads.go @@ -92,7 +92,7 @@ func (t *gcsFuseCSIWorkloadsTestSuite) DefineTests(driver storageframework.TestD ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand(fmt.Sprintf("mount | grep %v | grep rw, && echo 'hello world' > %v/${POD_NAME} && grep 'hello world' %v/${POD_NAME} && tail -f /dev/null", mountPath, mountPath, mountPath)) ginkgo.By("Configuring the deployment") @@ -113,7 +113,7 @@ func (t *gcsFuseCSIWorkloadsTestSuite) DefineTests(driver storageframework.TestD ginkgo.By("Configuring the pod") tPod1 := specs.NewTestPod(f.ClientSet, f.Namespace) tPod1.SetGracePeriod(600) - tPod1.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod1.SetupVolume(l.volumeResource, volumeName, mountPath, false) cmd := []string{ fmt.Sprintf("handle_term() { echo 'Caught SIGTERM signal!'; echo 'hello world' > %v/data && grep 'hello world' %v/data; sleep 5; exit 0; };", mountPath, mountPath), "trap handle_term SIGTERM;", @@ -141,7 +141,7 @@ func (t *gcsFuseCSIWorkloadsTestSuite) DefineTests(driver storageframework.TestD ginkgo.By("Configuring the second pod") tPod2 := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod2.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod2.SetupVolume(l.volumeResource, volumeName, mountPath, false) ginkgo.By("Deploying the second pod") tPod2.Create(ctx) @@ -161,7 +161,7 @@ func (t *gcsFuseCSIWorkloadsTestSuite) DefineTests(driver storageframework.TestD ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand(fmt.Sprintf("mount | grep %v | grep rw, && echo 'hello world' > %v/${POD_NAME} && grep 'hello world' %v/${POD_NAME} && tail -f /dev/null", mountPath, mountPath, mountPath)) ginkgo.By("Configuring the StatefulSet") @@ -182,7 +182,7 @@ func (t *gcsFuseCSIWorkloadsTestSuite) DefineTests(driver storageframework.TestD ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetRestartPolicy(v1.RestartPolicyNever) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) tPod.SetCommand("touch /mnt/test/test_file && echo $(date) >> /mnt/test/test_file && cat /mnt/test/test_file") ginkgo.By("Configuring the job") @@ -203,7 +203,7 @@ func (t *gcsFuseCSIWorkloadsTestSuite) DefineTests(driver storageframework.TestD ginkgo.By("Configuring the pod") tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetRestartPolicy(v1.RestartPolicyOnFailure) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) cmd := []string{ "sleep 10;", fmt.Sprintf("if [ -f %v/testfile ]; then echo 'Completed Successfully!'; exit 0; fi;", mountPath), @@ -232,7 +232,7 @@ func (t *gcsFuseCSIWorkloadsTestSuite) DefineTests(driver storageframework.TestD tPod := specs.NewTestPod(f.ClientSet, f.Namespace) tPod.SetGracePeriod(600) tPod.SetRestartPolicy(v1.RestartPolicyOnFailure) - tPod.SetupVolume(l.volumeResource, "test-gcsfuse-volume", mountPath, false) + tPod.SetupVolume(l.volumeResource, volumeName, mountPath, false) cmd := []string{ "sleep 10;", "exit 1;",