From 6a879ab4f4258719d97fb25141c6b0206080722c Mon Sep 17 00:00:00 2001 From: Gaius Date: Fri, 14 Feb 2025 12:08:42 +0800 Subject: [PATCH] test(e2e/v2): add test cases for dfcache (#3833) Signed-off-by: Gaius --- test/e2e/v2/dfcache_test.go | 308 ++++++++++++++++++++++++++++++++++++ test/e2e/v2/util/task.go | 28 ++++ 2 files changed, 336 insertions(+) create mode 100644 test/e2e/v2/dfcache_test.go diff --git a/test/e2e/v2/dfcache_test.go b/test/e2e/v2/dfcache_test.go new file mode 100644 index 00000000000..558d1f22748 --- /dev/null +++ b/test/e2e/v2/dfcache_test.go @@ -0,0 +1,308 @@ +/* + * Copyright 2025 The Dragonfly Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" //nolint + . "github.com/onsi/gomega" //nolint + + "d7y.io/dragonfly/v2/test/e2e/v2/util" +) + +var _ = Describe("Import and Export Using Dfcache", func() { + Context("1MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("import and export should be ok", Label("dfcache", "import and export"), func() { + var clientPod *util.PodExec + clientPod, err = util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + _, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput() + fmt.Println(string(importOut), err) + Expect(err).NotTo(HaveOccurred()) + + seedClientPod, err := util.SeedClientExec(0) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput() + fmt.Println(string(exportOut), err) + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + }) + }) + + Context("10MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("import and export should be ok", Label("dfcache", "import and export"), func() { + var clientPod *util.PodExec + clientPod, err = util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + _, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput() + fmt.Println(string(importOut), err) + Expect(err).NotTo(HaveOccurred()) + + seedClientPod, err := util.SeedClientExec(0) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput() + fmt.Println(string(exportOut), err) + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + }) + }) + + Context("100MiB file", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize100MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("import and export should be ok", Label("dfcache", "import and export"), func() { + var clientPod *util.PodExec + clientPod, err = util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + _, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput() + fmt.Println(string(importOut), err) + Expect(err).NotTo(HaveOccurred()) + + seedClientPod, err := util.SeedClientExec(0) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput() + fmt.Println(string(exportOut), err) + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + }) + }) + + Context("1MiB file and set export transfer-from-dfdaemon", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize1MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("import and export should be ok", Label("dfcache", "import and export", "export transfer-from-dfdaemon"), func() { + var clientPod *util.PodExec + clientPod, err = util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + _, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput() + fmt.Println(string(importOut), err) + Expect(err).NotTo(HaveOccurred()) + + seedClientPod, err := util.SeedClientExec(0) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --transfer-from-dfdaemon --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput() + fmt.Println(string(exportOut), err) + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + }) + }) + + Context("10MiB file and set export transfer-from-dfdaemon", func() { + var ( + testFile *util.File + err error + ) + + BeforeEach(func() { + testFile, err = util.GetFileServer().GenerateFile(util.FileSize10MiB) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile).NotTo(BeNil()) + }) + + AfterEach(func() { + err = util.GetFileServer().DeleteFile(testFile.GetInfo()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("import and export should be ok", Label("dfcache", "import and export", "export transfer-from-dfdaemon"), func() { + var clientPod *util.PodExec + clientPod, err = util.ClientExec() + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + _, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfget %s --disable-back-to-source --output %s", testFile.GetDownloadURL(), testFile.GetOutputPath())).CombinedOutput() + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err := util.CalculateSha256ByTaskID([]*util.PodExec{clientPod}, testFile.GetTaskID()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{clientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + importOut, err := clientPod.Command("sh", "-c", fmt.Sprintf("dfcache import %s --id %s", testFile.GetOutputPath(), testFile.GetSha256())).CombinedOutput() + fmt.Println(string(importOut), err) + Expect(err).NotTo(HaveOccurred()) + + seedClientPod, err := util.SeedClientExec(0) + fmt.Println(err) + Expect(err).NotTo(HaveOccurred()) + + exportOut, err := seedClientPod.Command("sh", "-c", fmt.Sprintf("dfcache export %s --transfer-from-dfdaemon --output %s", testFile.GetSha256(), testFile.GetOutputPath())).CombinedOutput() + fmt.Println(string(exportOut), err) + Expect(err).NotTo(HaveOccurred()) + + sha256sum, err = util.CalculateSha256ByPersistentCacheTaskID([]*util.PodExec{seedClientPod}, testFile.GetSha256()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + + sha256sum, err = util.CalculateSha256ByOutput([]*util.PodExec{seedClientPod}, testFile.GetOutputPath()) + Expect(err).NotTo(HaveOccurred()) + Expect(testFile.GetSha256()).To(Equal(sha256sum)) + }) + }) +}) diff --git a/test/e2e/v2/util/task.go b/test/e2e/v2/util/task.go index 71a1f3f9994..0793b0c59b3 100644 --- a/test/e2e/v2/util/task.go +++ b/test/e2e/v2/util/task.go @@ -73,6 +73,34 @@ func CalculateSha256ByTaskID(pods []*PodExec, taskID string) (string, error) { return sha256sum, nil } +func CalculateSha256ByPersistentCacheTaskID(pods []*PodExec, taskID string) (string, error) { + var sha256sum string + for _, pod := range pods { + contentPath := fmt.Sprintf("%s/persistent-cache-tasks/%s/%s", clientContentDir, taskID[:3], taskID) + if _, err := pod.Command("ls", contentPath).CombinedOutput(); err != nil { + // If the path does not exist, skip this client. + fmt.Printf("path %s does not exist: %s\n", contentPath, err.Error()) + continue + } + + // Calculate sha256sum of the task content. + out, err := pod.Command("sh", "-c", fmt.Sprintf("sha256sum %s", contentPath)).CombinedOutput() + if err != nil { + return "", fmt.Errorf("calculate sha256sum of %s failed: %s", contentPath, err.Error()) + } + + fmt.Println("sha256sum: " + string(out)) + sha256sum = strings.Split(string(out), " ")[0] + break + } + + if sha256sum == "" { + return "", errors.New("can not found sha256sum") + } + + return sha256sum, nil +} + func CalculateSha256ByOutput(pods []*PodExec, output string) (string, error) { var sha256sum string for _, pod := range pods {