From ff867ba9b7ae5d1b96052eb5980b01851817842b Mon Sep 17 00:00:00 2001 From: "suhan.zcy" Date: Thu, 2 Jan 2025 11:00:54 +0800 Subject: [PATCH] refactor: refactor the e2e file server Signed-off-by: suhan.zcy --- .github/workflows/compatibility-e2e-v2.yml | 2 + .github/workflows/e2e-v2.yml | 4 +- test/e2e/v2/concurrency_test.go | 76 ++++++------- test/e2e/v2/e2e_test.go | 13 +++ test/e2e/v2/util/file.go | 60 ++++++++++ test/e2e/v2/util/file_server_v2.go | 126 +++++++++++++++++++++ test/e2e/v2/util/options.go | 85 ++++++++++++++ test/testdata/k8s/file-server-v2.yaml | 56 +++++++++ 8 files changed, 381 insertions(+), 41 deletions(-) create mode 100644 test/e2e/v2/util/file.go create mode 100644 test/e2e/v2/util/file_server_v2.go create mode 100644 test/e2e/v2/util/options.go create mode 100644 test/testdata/k8s/file-server-v2.yaml diff --git a/.github/workflows/compatibility-e2e-v2.yml b/.github/workflows/compatibility-e2e-v2.yml index d8b220c8a15..ca276fc585a 100644 --- a/.github/workflows/compatibility-e2e-v2.yml +++ b/.github/workflows/compatibility-e2e-v2.yml @@ -20,6 +20,7 @@ env: DRAGONFLY_CHARTS_CONFIG_PATH: test/testdata/charts/config-v2.yaml DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml + DRAGONFLY_FILE_SERVER_V2_PATH: test/testdata/k8s/file-server-v2.yaml jobs: compatibility_e2e_tests: @@ -137,6 +138,7 @@ jobs: run: | helm install --wait --timeout 15m --dependency-update --create-namespace --namespace dragonfly-system --set ${{ matrix.chart-name }}.image.tag=${{ matrix.image-tag }} --set ${{ matrix.chart-name }}.image.repository=dragonflyoss/${{ matrix.image }} -f ${{ env.DRAGONFLY_CHARTS_CONFIG_PATH }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} + kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_V2_PATH }} kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m - name: Run E2E test diff --git a/.github/workflows/e2e-v2.yml b/.github/workflows/e2e-v2.yml index 5385f03026d..1b0172680f0 100644 --- a/.github/workflows/e2e-v2.yml +++ b/.github/workflows/e2e-v2.yml @@ -10,7 +10,7 @@ on: schedule: - cron: '0 4 * * *' -permissions: +permissions: contents: read env: @@ -19,6 +19,7 @@ env: KIND_CONFIG_PATH: test/testdata/kind/config-v2.yaml DRAGONFLY_CHARTS_PATH: deploy/helm-charts/charts/dragonfly DRAGONFLY_FILE_SERVER_PATH: test/testdata/k8s/file-server.yaml + DRAGONFLY_FILE_SERVER_V2_PATH: test/testdata/k8s/file-server-v2.yaml jobs: e2e_tests: @@ -124,6 +125,7 @@ jobs: run: | helm install --wait --timeout 15m --dependency-update --create-namespace --namespace dragonfly-system -f ${{ matrix.charts-config }} dragonfly ${{ env.DRAGONFLY_CHARTS_PATH }} kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_PATH }} + kubectl apply -f ${{ env.DRAGONFLY_FILE_SERVER_V2_PATH }} kubectl wait po file-server-0 --namespace dragonfly-e2e --for=condition=ready --timeout=10m - name: Run E2E test diff --git a/test/e2e/v2/concurrency_test.go b/test/e2e/v2/concurrency_test.go index 4a5cbd6e2c1..b269d4e5d5f 100644 --- a/test/e2e/v2/concurrency_test.go +++ b/test/e2e/v2/concurrency_test.go @@ -27,23 +27,34 @@ import ( var _ = Describe("Download Concurrency", func() { Context("ab", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = fs.PrepareFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = fs.CleanFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + It("concurrent 100 should be ok", Label("concurrent", "100"), func() { clientPod, err := util.ClientExec() fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 100 -n 200 -X 127.0.0.1:4001 %s", util.GetFileURL("/bin/unshare"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 100 -n 200 -X 127.0.0.1:4001 %s", testFile.GetDownloadURL())).CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "14b31801ea6990788057b965fbc51e44bf73800462915fdfa0fda8182acca4d6", - Sha256: "fc44bbbba20490450c73530db3d1b935f893f38d7d8084ca132952a765ff5ff6", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { @@ -52,9 +63,9 @@ var _ = Describe("Download Concurrency", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) It("concurrent 200 should be ok", Label("concurrent", "200"), func() { @@ -62,18 +73,13 @@ var _ = Describe("Download Concurrency", func() { fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 200 -n 400 -X 127.0.0.1:4001 %s", util.GetFileURL("/bin/loginctl"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 200 -n 400 -X 127.0.0.1:4001 %s", testFile.GetDownloadURL())).CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "958e177b56be708c9d7ec193ae8cef399b39faff8234af33efa4cbe097d1fc5f", - Sha256: "dc102987a36be20846821ac74648534863ff0fe8897d4250273a6ffc80481d91", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { @@ -82,9 +88,9 @@ var _ = Describe("Download Concurrency", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) It("concurrent 500 should be ok", Label("concurrent", "500"), func() { @@ -92,18 +98,13 @@ var _ = Describe("Download Concurrency", func() { fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 500 -n 1000 -X 127.0.0.1:4001 %s", util.GetFileURL("/bin/realpath"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 500 -n 1000 -X 127.0.0.1:4001 %s", testFile.GetDownloadURL())).CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "dd573cf9c3e1a79402b8423abcd1ba987c1b1ee9c49069d139d71106a260b055", - Sha256: "54e54b7ff54ef70d4db2adcd24a27e3b9af3cd99fc0213983bac1e8035429be6", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { @@ -112,9 +113,9 @@ var _ = Describe("Download Concurrency", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) It("concurrent 1000 should be ok", Label("concurrent", "1000"), func() { @@ -122,18 +123,13 @@ var _ = Describe("Download Concurrency", func() { fmt.Println(err) Expect(err).NotTo(HaveOccurred()) - out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 1000 -n 2000 -X 127.0.0.1:4001 %s", util.GetFileURL("/bin/lnstat"))).CombinedOutput() + out, err := clientPod.Command("sh", "-c", fmt.Sprintf("ab -c 1000 -n 2000 -X 127.0.0.1:4001 %s", testFile.GetDownloadURL())).CombinedOutput() fmt.Println(string(out)) Expect(err).NotTo(HaveOccurred()) - fileMetadata := util.FileMetadata{ - ID: "f1957adc26ec326800ced850d72e583a03be0999ba80d9aa2e3ba57ef4ddaf17", - Sha256: "87c09b7c338f258809ca2d436bbe06ac94a3166b3f3e1125a86f35d9a9aa1d2f", - } - - sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, fileMetadata.ID) + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) seedClientPods := make([]*util.PodExec, 3) for i := 0; i < 3; i++ { @@ -142,9 +138,9 @@ var _ = Describe("Download Concurrency", func() { Expect(err).NotTo(HaveOccurred()) } - sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, fileMetadata.ID) + sha256sum, err = util.CalculateSha256ByTaskID(seedClientPods, testFile.GetTaskID()) Expect(err).NotTo(HaveOccurred()) - Expect(fileMetadata.Sha256).To(Equal(sha256sum)) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) }) }) }) diff --git a/test/e2e/v2/e2e_test.go b/test/e2e/v2/e2e_test.go index 0cad14fd471..fabf47a34a0 100644 --- a/test/e2e/v2/e2e_test.go +++ b/test/e2e/v2/e2e_test.go @@ -30,6 +30,9 @@ import ( "d7y.io/dragonfly/v2/test/e2e/v2/util" ) +// fs is the global file server. +var fs *util.FileServer + var _ = AfterSuite(func() { for _, server := range util.Servers { for i := 0; i < server.Replicas; i++ { @@ -80,9 +83,19 @@ var _ = AfterSuite(func() { fmt.Printf("------------------------------ Get %s-%d Artifact Finished ------------------------------\n", server.Name, i) } } + + // Clean up file server if exists. + if fs != nil { + fs.CleanAll() + } }) var _ = BeforeSuite(func() { + var err error + fs, err = util.NewFileServer() + Expect(err).NotTo(HaveOccurred()) + Expect(fs).NotTo(BeNil()) + rawGitCommit, err := util.GitCommand("rev-parse", "--short", "HEAD").CombinedOutput() Expect(err).NotTo(HaveOccurred()) gitCommit := strings.Fields(string(rawGitCommit))[0] diff --git a/test/e2e/v2/util/file.go b/test/e2e/v2/util/file.go new file mode 100644 index 00000000000..3915c4b2433 --- /dev/null +++ b/test/e2e/v2/util/file.go @@ -0,0 +1,60 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "os" + + "d7y.io/dragonfly/v2/pkg/idgen" +) + +// File represents a file. +type File struct { + // info is the local file info. + info os.FileInfo + // downloadURL is the download URL of the file from remote file server. + downloadURL string +} + +// GetInfo returns the file info. +func (f *File) GetInfo() os.FileInfo { + return f.info +} + +// GetSha256 returns the sha256 of the file content. +func (f *File) GetSha256() string { + // the file name is the sha256 of the file content + return f.info.Name() +} + +// GetDownloadURL returns the download URL of the file from remote file server. +func (f *File) GetDownloadURL() string { + return f.downloadURL +} + +// GetTaskID returns the task id of the file. +func (f *File) GetTaskID(opts ...TaskIDOption) string { + taskIDOpts := &TaskIDOptions{ + url: f.downloadURL, + } + + for _, opt := range opts { + opt(taskIDOpts) + } + + return idgen.TaskIDV2(taskIDOpts.url, taskIDOpts.tag, taskIDOpts.application, taskIDOpts.filteredQueryParams) +} diff --git a/test/e2e/v2/util/file_server_v2.go b/test/e2e/v2/util/file_server_v2.go new file mode 100644 index 00000000000..d8da7206447 --- /dev/null +++ b/test/e2e/v2/util/file_server_v2.go @@ -0,0 +1,126 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/url" + "os" + "path/filepath" +) + +type FileSize = uint64 + +const ( + // FileSize1MiB represents the size of 1MiB. + FileSize1MiB FileSize = 1024 * 1024 + // FileSize10MiB represents the size of 10MiB. + FileSize10MiB FileSize = 10 * FileSize1MiB + // FileSize100MiB represents the size of 100MiB. + FileSize100MiB FileSize = 100 * FileSize1MiB +) + +const ( + // defaultFileServerEndpoint is the default endpoint of the file server. + defaultFileServerEndpoint = "http://file-server-v2.dragonfly-e2e.svc" + // defaultFileServerLocalDir is the default local directory of the file server. + defaultFileServerLocalDir = "/data/e2e-file-server" +) + +// FileServer represents a file server. +type FileServer struct { + // endpoint is the endpoint of the file server. + endpoint *url.URL + // localDir is the local directory of the file server. + localDir string +} + +// NewFileServer creates a new file server. +func NewFileServer(opts ...FileServerOption) (*FileServer, error) { + fileServerOptions := &FileServerOptions{ + endpoint: defaultFileServerEndpoint, + localDir: defaultFileServerLocalDir, + } + + for _, opt := range opts { + opt(fileServerOptions) + } + + // parse the endpoint + var u *url.URL + u, err := url.Parse(fileServerOptions.endpoint) + if err != nil { + return nil, err + } + + // ensure the localDir exists + if err = os.MkdirAll(fileServerOptions.localDir, 0755); err != nil { + return nil, err + } + + fileServer := &FileServer{ + endpoint: u, + localDir: fileServerOptions.localDir, + } + + return fileServer, nil +} + +// PrepareFile prepares a file by the size and returns File instance. +func (fs *FileServer) PrepareFile(size FileSize) (*File, error) { + // create file in the localDir + data := make([]byte, size) + _, err := rand.Read(data) + if err != nil { + return nil, err + } + + hash := sha256.Sum256(data) + fileName := hex.EncodeToString(hash[:]) + filePath := filepath.Join(fs.localDir, fileName) + if err = os.WriteFile(filePath, data, 0644); err != nil { + return nil, err + } + + // get file info + info, err := os.Stat(filePath) + if err != nil { + return nil, err + } + + file := &File{ + info: info, + downloadURL: fmt.Sprintf("%s/%s", fs.endpoint.String(), fileName), + } + + return file, nil +} + +// CleanFile removes the file by the file info. +func (fs *FileServer) CleanFile(info os.FileInfo) error { + // remove the file in the local dir + filepath := filepath.Join(fs.localDir, info.Name()) + return os.RemoveAll(filepath) +} + +// CleanAll removes all files in the local dir. +func (fs *FileServer) CleanAll() error { + return os.RemoveAll(fs.localDir) +} diff --git a/test/e2e/v2/util/options.go b/test/e2e/v2/util/options.go new file mode 100644 index 00000000000..b4511742367 --- /dev/null +++ b/test/e2e/v2/util/options.go @@ -0,0 +1,85 @@ +/* + * Copyright 2024 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package util + +// FileServerOptions represents the options of the file server. +type FileServerOptions struct { + // endpoint is the endpoint of the file server. + endpoint string + // localDir is the local temporary directory of the file server. + localDir string +} + +// FileServerOption is the type of the options of the file server. +type FileServerOption func(*FileServerOptions) + +// WithFileServerEndpoint sets the endpoint of the file server. +func WithFileServerEndpoint(endpoint string) FileServerOption { + return func(o *FileServerOptions) { + o.endpoint = endpoint + } +} + +// WithFileServerLocalDir sets the local temporary directory of the file server. +func WithFileServerLocalDir(localDir string) FileServerOption { + return func(o *FileServerOptions) { + o.localDir = localDir + } +} + +// TaskIDOptions represents the options of the task id. +type TaskIDOptions struct { + // url is the url of the download task. + url string + // tag is the tag of the download task. + tag string + // appliccation is the application of the download task. + application string + // filteredQueryParams is the filtered query params of the download task. + filteredQueryParams []string +} + +// TaskIDOption is the type of the options of the task id. +type TaskIDOption func(*TaskIDOptions) + +// WithTaskIDURL sets the url of the download task. +func WithTaskIDURL(url string) TaskIDOption { + return func(o *TaskIDOptions) { + o.url = url + } +} + +// WithTaskIDTag sets the tag of the download task. +func WithTaskIDTag(tag string) TaskIDOption { + return func(o *TaskIDOptions) { + o.tag = tag + } +} + +// WithTaskIDApplication sets the application of the download task. +func WithTaskIDApplication(application string) TaskIDOption { + return func(o *TaskIDOptions) { + o.application = application + } +} + +// WithTaskIDFilteredQueryParams sets the filtered query params of the download task. +func WithTaskIDFilteredQueryParams(filteredQueryParams []string) TaskIDOption { + return func(o *TaskIDOptions) { + o.filteredQueryParams = filteredQueryParams + } +} diff --git a/test/testdata/k8s/file-server-v2.yaml b/test/testdata/k8s/file-server-v2.yaml new file mode 100644 index 00000000000..4cb290eb444 --- /dev/null +++ b/test/testdata/k8s/file-server-v2.yaml @@ -0,0 +1,56 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: dragonfly-e2e + +--- +apiVersion: v1 +kind: Service +metadata: + name: file-server-v2 + namespace: dragonfly-e2e +spec: + selector: + app: dragonfly + component: file-server-v2 + type: ClusterIP + ports: + - name: server + port: 80 + protocol: TCP + targetPort: 5000 + +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: file-server-v2 + namespace: dragonfly-e2e +spec: + serviceName: file-server-v2 + selector: + matchLabels: + app: dragonfly + component: file-server-v2 + replicas: 1 + template: + metadata: + labels: + app: dragonfly + component: file-server-v2 + spec: + containers: + - name: dufs + image: sigoden/dufs + command: ["dufs","/data","-A"] + imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 5000 + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + hostPath: + path: /data/e2e-file-server