From 05a9c692e0667d5574c9942ca0c8f9c0dd320974 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Tue, 7 Jun 2022 16:46:16 +0200 Subject: [PATCH 01/13] If applied, this commit will add options including recursive functionality to FileClient().Get(..) Signed-off-by: Rana Tarek Hassan --- github/client_repository_file.go | 24 ++++++++++++++- github/integration_test.go | 53 ++++++++++++++++++++++++++++++++ gitlab/client_repository_file.go | 15 +++++++-- gitlab/integration_test.go | 53 ++++++++++++++++++++++++++++++++ gitprovider/client.go | 2 +- gitprovider/options.go | 17 ++++++++++ stash/client_repository_file.go | 2 +- 7 files changed, 160 insertions(+), 6 deletions(-) diff --git a/github/client_repository_file.go b/github/client_repository_file.go index b77cd055..d9d4d7e1 100644 --- a/github/client_repository_file.go +++ b/github/client_repository_file.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "io/ioutil" + "strings" "github.com/fluxcd/go-git-providers/gitprovider" "github.com/google/go-github/v45/github" @@ -35,12 +36,17 @@ type FileClient struct { } // Get fetches and returns the contents of a file from a given branch and path -func (c *FileClient) Get(ctx context.Context, path, branch string) ([]*gitprovider.CommitFile, error) { +func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) { opts := &github.RepositoryContentGetOptions{ Ref: branch, } + recursive := false + for _, opt := range optFns { + recursive = opt.ApplyFilesGetOptions(&gitprovider.FilesGetOptions{}).Recursive + } + _, directoryContent, _, err := c.c.Client().Repositories.GetContents(ctx, c.ref.GetIdentity(), c.ref.GetRepository(), path, opts) if err != nil { return nil, err @@ -54,6 +60,22 @@ func (c *FileClient) Get(ctx context.Context, path, branch string) ([]*gitprovid for _, file := range directoryContent { filePath := file.Path + if *file.Type == "dir" { + if recursive == true { + if !strings.HasSuffix(path, "/") { + path = path + "/" + } + subdirectoryPath := fmt.Sprintf("%v%v/", path, *file.Name) + // recursive call for child directories to get their content + childFiles, err := c.Get(ctx, subdirectoryPath, branch, optFns...) + if err != nil { + return nil, err + } + files = append(files, childFiles...) + } + continue + + } output, _, err := c.c.Client().Repositories.DownloadContents(ctx, c.ref.GetIdentity(), c.ref.GetRepository(), *filePath, opts) if err != nil { return nil, err diff --git a/github/integration_test.go b/github/integration_test.go index 05d8c599..289c19fa 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -562,6 +562,59 @@ var _ = Describe("GitHub Provider", func() { }) + It("should be possible to download files from path and branch specified with nested directory", func() { + + userRepoRef := newUserRepoRef(testUser, testUserRepoName) + + userRepo, err := c.UserRepositories().Get(ctx, userRepoRef) + Expect(err).ToNot(HaveOccurred()) + + defaultBranch := userRepo.Get().DefaultBranch + + path0 := "clustersDir/cluster/machine.yaml" + content0 := "machine yaml content" + path1 := "clustersDir/cluster/machine1.yaml" + content1 := "machine1 yaml content" + path2 := "clustersDir/cluster2/clusterSubDir/machine2.yaml" + content2 := "machine2 yaml content" + + files := []gitprovider.CommitFile{ + { + Path: &path0, + Content: &content0, + }, + { + Path: &path1, + Content: &content1, + }, + { + Path: &path2, + Content: &content2, + }, + } + + commitFiles := make([]gitprovider.CommitFile, 0) + for _, file := range files { + path := file.Path + content := file.Content + commitFiles = append(commitFiles, gitprovider.CommitFile{ + Path: path, + Content: content, + }) + } + + _, err = userRepo.Commits().Create(ctx, *defaultBranch, "added config files", commitFiles) + Expect(err).ToNot(HaveOccurred()) + + options := gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true)} + downloadedFiles, err := userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) + Expect(err).ToNot(HaveOccurred()) + for ind, downloadedFile := range downloadedFiles { + Expect(*downloadedFile).To(Equal(files[ind])) + } + + }) + AfterSuite(func() { if os.Getenv("SKIP_CLEANUP") == "1" { return diff --git a/gitlab/client_repository_file.go b/gitlab/client_repository_file.go index d93043b4..ce4e9dd2 100644 --- a/gitlab/client_repository_file.go +++ b/gitlab/client_repository_file.go @@ -36,11 +36,17 @@ type FileClient struct { } // Get fetches and returns the contents of a file from a given branch and path -func (c *FileClient) Get(_ context.Context, path, branch string) ([]*gitprovider.CommitFile, error) { +func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) { + + recursive := false + for _, opt := range optFns { + recursive = opt.ApplyFilesGetOptions(&gitprovider.FilesGetOptions{}).Recursive + } opts := &gitlab.ListTreeOptions{ - Path: &path, - Ref: &branch, + Path: &path, + Ref: &branch, + Recursive: &recursive, } listFiles, _, err := c.c.Client().Repositories.ListTree(getRepoPath(c.ref), opts) @@ -54,6 +60,9 @@ func (c *FileClient) Get(_ context.Context, path, branch string) ([]*gitprovider files := make([]*gitprovider.CommitFile, 0) for _, file := range listFiles { + if file.Type == "tree" { + continue + } fileDownloaded, _, err := c.c.Client().RepositoryFiles.GetFile(getRepoPath(c.ref), file.Path, fileOpts) if err != nil { return nil, err diff --git a/gitlab/integration_test.go b/gitlab/integration_test.go index 96f3693d..baaf9224 100644 --- a/gitlab/integration_test.go +++ b/gitlab/integration_test.go @@ -886,6 +886,59 @@ var _ = Describe("GitLab Provider", func() { }) + It("should be possible to download files from path and branch specified with nested directory", func() { + + userRepoRef := newUserRepoRef(testUserName, testRepoName) + + userRepo, err := c.UserRepositories().Get(ctx, userRepoRef) + Expect(err).ToNot(HaveOccurred()) + + defaultBranch := userRepo.Get().DefaultBranch + + path0 := "clustersDir/cluster/machine.yaml" + content0 := "machine0 yaml content" + path1 := "clustersDir/cluster/machine1.yaml" + content1 := "machine1 yaml content" + path2 := "clustersDir/cluster2/clusterSubDir/machine2.yaml" + content2 := "machine2 yaml content" + + files := []gitprovider.CommitFile{ + { + Path: &path0, + Content: &content0, + }, + { + Path: &path1, + Content: &content1, + }, + { + Path: &path2, + Content: &content2, + }, + } + + commitFiles := make([]gitprovider.CommitFile, 0) + for _, file := range files { + path := file.Path + content := file.Content + commitFiles = append(commitFiles, gitprovider.CommitFile{ + Path: path, + Content: content, + }) + } + + _, err = userRepo.Commits().Create(ctx, *defaultBranch, "added files", commitFiles) + Expect(err).ToNot(HaveOccurred()) + + options := gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true)} + downloadedFiles, err := userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) + Expect(err).ToNot(HaveOccurred()) + for ind, downloadedFile := range downloadedFiles { + Expect(*downloadedFile).To(Equal(files[ind])) + } + + }) + AfterSuite(func() { if os.Getenv("SKIP_CLEANUP") == "1" { return diff --git a/gitprovider/client.go b/gitprovider/client.go index b95720d7..55726b8e 100644 --- a/gitprovider/client.go +++ b/gitprovider/client.go @@ -241,5 +241,5 @@ type PullRequestClient interface { // This client can be accessed through Repository.Branches(). type FileClient interface { // GetFiles fetch files content from specific path and branch - Get(ctx context.Context, path, branch string) ([]*CommitFile, error) + Get(ctx context.Context, path, branch string, optFns ...FilesGetOption) ([]*CommitFile, error) } diff --git a/gitprovider/options.go b/gitprovider/options.go index 8109e334..02072307 100644 --- a/gitprovider/options.go +++ b/gitprovider/options.go @@ -77,3 +77,20 @@ func (opts *RepositoryCreateOptions) ValidateOptions() error { } return errs.Error() } + +// FilesGetOptions specifies optional options when fetcing files. +type FilesGetOptions struct { + Recursive bool +} + +type FilesGetOption interface { + ApplyFilesGetOptions(target *FilesGetOptions) FilesGetOptions +} + +func (opts *FilesGetOptions) ApplyFilesGetOptions(target *FilesGetOptions) FilesGetOptions { + // Go through each field in opts, and apply it to target if set + if opts.Recursive == true { + target.Recursive = opts.Recursive + } + return *target +} diff --git a/stash/client_repository_file.go b/stash/client_repository_file.go index 72405a48..66572d64 100644 --- a/stash/client_repository_file.go +++ b/stash/client_repository_file.go @@ -33,6 +33,6 @@ type FileClient struct { } // Get fetches and returns the contents of a file from a given branch and path -func (c *FileClient) Get(_ context.Context, path, branch string) ([]*gitprovider.CommitFile, error) { +func (c *FileClient) Get(_ context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) { return nil, fmt.Errorf("error getting file %s@%s. not implemented in stash yet", path, branch) } From e8a44aac455c31f6a9d4842fe3425dec142386e9 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Wed, 15 Jun 2022 12:18:04 +0200 Subject: [PATCH 02/13] If applied, this commit will add maxDepth and maxFiles options to File Client Get Signed-off-by: Rana Tarek Hassan --- github/client_repository_file.go | 37 ++++++++++++++++++++++++++------ github/integration_test.go | 24 ++++++++++++++++++++- gitlab/client_repository_file.go | 7 +++--- gitprovider/options.go | 13 ++++++++--- 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/github/client_repository_file.go b/github/client_repository_file.go index d9d4d7e1..4b6abaad 100644 --- a/github/client_repository_file.go +++ b/github/client_repository_file.go @@ -35,16 +35,29 @@ type FileClient struct { ref gitprovider.RepositoryRef } -// Get fetches and returns the contents of a file from a given branch and path +// if number of files retrieved exceed the max number of files, return only until the number of maxFiles +func limitSliceSize(s []*gitprovider.CommitFile, maxLength int) []*gitprovider.CommitFile { + if maxLength != 0 && len(s) > maxLength { + return s[:maxLength] + } + return s +} + +// Get fetches and returns the contents of a file or directory from a given branch and path with possible options of FilesGetOption +// If recursive option is provided, the files are retrieved recursively from subdirectories of the base path. +// If recursive and MaxDepth options are provided, the files rare etrieved recursively from subdirectories until reaching the max depth of levels +// If maxFiles option is provided, the number of files are maximized to the number provided + +// Uses https://docs.github.com/en/rest/repos/contents#get-repository-content func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) { opts := &github.RepositoryContentGetOptions{ Ref: branch, } - recursive := false + fileOpts := gitprovider.FilesGetOptions{} for _, opt := range optFns { - recursive = opt.ApplyFilesGetOptions(&gitprovider.FilesGetOptions{}).Recursive + opt.ApplyFilesGetOptions(&fileOpts) } _, directoryContent, _, err := c.c.Client().Repositories.GetContents(ctx, c.ref.GetIdentity(), c.ref.GetRepository(), path, opts) @@ -61,17 +74,26 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git for _, file := range directoryContent { filePath := file.Path if *file.Type == "dir" { - if recursive == true { + if fileOpts.Recursive == true { + // stop recursive calls when level is the max level reached + if fileOpts.MaxDepth == 1 { + break + } + if !strings.HasSuffix(path, "/") { path = path + "/" } - subdirectoryPath := fmt.Sprintf("%v%v/", path, *file.Name) + subdirectoryPath := fmt.Sprintf("%s%s/", path, *file.Name) + // recursive call for child directories to get their content - childFiles, err := c.Get(ctx, subdirectoryPath, branch, optFns...) + childOptFns := gitprovider.FilesGetOptions{Recursive: fileOpts.Recursive, MaxFiles: fileOpts.MaxFiles, MaxDepth: fileOpts.MaxDepth - 1} + childFiles, err := c.Get(ctx, subdirectoryPath, branch, &childOptFns) + if err != nil { return nil, err } files = append(files, childFiles...) + files = limitSliceSize(files, fileOpts.MaxFiles) } continue @@ -93,7 +115,8 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git Path: filePath, Content: &contentStr, }) + } - return files, nil + return limitSliceSize(files, fileOpts.MaxFiles), nil } diff --git a/github/integration_test.go b/github/integration_test.go index 289c19fa..6815999c 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -562,7 +562,7 @@ var _ = Describe("GitHub Provider", func() { }) - It("should be possible to download files from path and branch specified with nested directory", func() { + It("should be possible to download files from path and branch specified with nested directory and max options", func() { userRepoRef := newUserRepoRef(testUser, testUserRepoName) @@ -606,6 +606,7 @@ var _ = Describe("GitHub Provider", func() { _, err = userRepo.Commits().Create(ctx, *defaultBranch, "added config files", commitFiles) Expect(err).ToNot(HaveOccurred()) + //Recursive option check options := gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true)} downloadedFiles, err := userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) Expect(err).ToNot(HaveOccurred()) @@ -613,6 +614,27 @@ var _ = Describe("GitHub Provider", func() { Expect(*downloadedFile).To(Equal(files[ind])) } + //Max depth option check + //should be files in 2 levels: machine.yaml and machine1.yaml in level 2 (level 1 has cluster and cluster2 dirs) + options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 2} + downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) + Expect(err).ToNot(HaveOccurred()) + Expect(downloadedFiles).To(HaveLen(2)) + + // Max files option check with recursion + // should be 1 file :machine.yaml + options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxFiles: 1} + downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) + Expect(err).ToNot(HaveOccurred()) + Expect(downloadedFiles).To(HaveLen(1)) + + // Max files option check with no recursion (base level) + // should be 0 + options = gitprovider.FilesGetOptions{MaxFiles: 2} + downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) + Expect(err).ToNot(HaveOccurred()) + Expect(downloadedFiles).To(HaveLen(0)) + }) AfterSuite(func() { diff --git a/gitlab/client_repository_file.go b/gitlab/client_repository_file.go index ce4e9dd2..73dd57c9 100644 --- a/gitlab/client_repository_file.go +++ b/gitlab/client_repository_file.go @@ -38,15 +38,16 @@ type FileClient struct { // Get fetches and returns the contents of a file from a given branch and path func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) { - recursive := false + filesGetOpts := gitprovider.FilesGetOptions{} + for _, opt := range optFns { - recursive = opt.ApplyFilesGetOptions(&gitprovider.FilesGetOptions{}).Recursive + opt.ApplyFilesGetOptions(&filesGetOpts) } opts := &gitlab.ListTreeOptions{ Path: &path, Ref: &branch, - Recursive: &recursive, + Recursive: &filesGetOpts.Recursive, } listFiles, _, err := c.c.Client().Repositories.ListTree(getRepoPath(c.ref), opts) diff --git a/gitprovider/options.go b/gitprovider/options.go index 02072307..de9bd48a 100644 --- a/gitprovider/options.go +++ b/gitprovider/options.go @@ -81,16 +81,23 @@ func (opts *RepositoryCreateOptions) ValidateOptions() error { // FilesGetOptions specifies optional options when fetcing files. type FilesGetOptions struct { Recursive bool + MaxFiles int + MaxDepth int } type FilesGetOption interface { - ApplyFilesGetOptions(target *FilesGetOptions) FilesGetOptions + ApplyFilesGetOptions(target *FilesGetOptions) } -func (opts *FilesGetOptions) ApplyFilesGetOptions(target *FilesGetOptions) FilesGetOptions { +func (opts *FilesGetOptions) ApplyFilesGetOptions(target *FilesGetOptions) { // Go through each field in opts, and apply it to target if set if opts.Recursive == true { target.Recursive = opts.Recursive } - return *target + if opts.MaxFiles > 0 { + target.MaxFiles = opts.MaxFiles + } + if opts.MaxDepth > 0 { + target.MaxDepth = opts.MaxDepth + } } From 5ad08497b23bbe8aa07a5ac0b5d096dec66a3631 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Wed, 15 Jun 2022 19:30:35 +0200 Subject: [PATCH 03/13] If applied, this commit will update changes for maxDepth,maxFiles, and recursive options in Files Get Signed-off-by: Rana Tarek Hassan --- github/client_repository_file.go | 33 ++++++++++++++++++++------------ github/integration_test.go | 2 +- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/github/client_repository_file.go b/github/client_repository_file.go index 4b6abaad..1b9fce8b 100644 --- a/github/client_repository_file.go +++ b/github/client_repository_file.go @@ -35,17 +35,18 @@ type FileClient struct { ref gitprovider.RepositoryRef } -// if number of files retrieved exceed the max number of files, return only until the number of maxFiles -func limitSliceSize(s []*gitprovider.CommitFile, maxLength int) []*gitprovider.CommitFile { - if maxLength != 0 && len(s) > maxLength { - return s[:maxLength] +// if number of files retrieved exceed the max number of files, return slice only until the number of maxFiles with bool true for limiting succeeded +func limitSliceSize(s []*gitprovider.CommitFile, size int) ([]*gitprovider.CommitFile, bool) { + if size != 0 && len(s) > size { + return s[:size], true } - return s + return s, false } // Get fetches and returns the contents of a file or directory from a given branch and path with possible options of FilesGetOption // If recursive option is provided, the files are retrieved recursively from subdirectories of the base path. -// If recursive and MaxDepth options are provided, the files rare etrieved recursively from subdirectories until reaching the max depth of levels +// If recursive and MaxDepth options are provided, the files are retrieved recursively from subdirectories until reaching the max depth of levels +// Recursive and MaxDepth should be provided together, default MaxDepth : 0 // If maxFiles option is provided, the number of files are maximized to the number provided // Uses https://docs.github.com/en/rest/repos/contents#get-repository-content @@ -74,10 +75,10 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git for _, file := range directoryContent { filePath := file.Path if *file.Type == "dir" { - if fileOpts.Recursive == true { + if fileOpts.Recursive { // stop recursive calls when level is the max level reached - if fileOpts.MaxDepth == 1 { - break + if fileOpts.MaxDepth <= 1 { + continue } if !strings.HasSuffix(path, "/") { @@ -86,14 +87,18 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git subdirectoryPath := fmt.Sprintf("%s%s/", path, *file.Name) // recursive call for child directories to get their content - childOptFns := gitprovider.FilesGetOptions{Recursive: fileOpts.Recursive, MaxFiles: fileOpts.MaxFiles, MaxDepth: fileOpts.MaxDepth - 1} + childMaxFiles := fileOpts.MaxFiles - len(files) + childOptFns := gitprovider.FilesGetOptions{Recursive: fileOpts.Recursive, MaxFiles: childMaxFiles, MaxDepth: fileOpts.MaxDepth - 1} childFiles, err := c.Get(ctx, subdirectoryPath, branch, &childOptFns) if err != nil { return nil, err } files = append(files, childFiles...) - files = limitSliceSize(files, fileOpts.MaxFiles) + files, limitSuccess := limitSliceSize(files, fileOpts.MaxFiles) + if limitSuccess { + return files, nil + } } continue @@ -115,8 +120,12 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git Path: filePath, Content: &contentStr, }) + files, limitSuccess := limitSliceSize(files, fileOpts.MaxFiles) + if limitSuccess { + return files, nil + } } - return limitSliceSize(files, fileOpts.MaxFiles), nil + return files, nil } diff --git a/github/integration_test.go b/github/integration_test.go index 6815999c..5bc7eaec 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -623,7 +623,7 @@ var _ = Describe("GitHub Provider", func() { // Max files option check with recursion // should be 1 file :machine.yaml - options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxFiles: 1} + options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 2, MaxFiles: 1} downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) Expect(err).ToNot(HaveOccurred()) Expect(downloadedFiles).To(HaveLen(1)) From deb853091b25e24a1d17209387b2379dbceea186 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Thu, 16 Jun 2022 14:06:11 +0200 Subject: [PATCH 04/13] If applied, this commit will add getDirectoryFiles helper to make recursive calls in Files Get Add more test cases for recursive, maxDepth, maxFiles options in Files Get Signed-off-by: Rana Tarek Hassan --- github/client_repository_file.go | 47 ++++++++++++++++++++------------ github/integration_test.go | 21 ++++++++++++++ 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/github/client_repository_file.go b/github/client_repository_file.go index 1b9fce8b..71f259c3 100644 --- a/github/client_repository_file.go +++ b/github/client_repository_file.go @@ -37,12 +37,34 @@ type FileClient struct { // if number of files retrieved exceed the max number of files, return slice only until the number of maxFiles with bool true for limiting succeeded func limitSliceSize(s []*gitprovider.CommitFile, size int) ([]*gitprovider.CommitFile, bool) { - if size != 0 && len(s) > size { + if size != 0 && len(s) >= size { return s[:size], true } return s, false } +// getDirectoryFiles will make the recursive call to get files in sub directory of a dir type +func (c *FileClient) getDirectoryFiles(ctx context.Context, path string, branch string, dir *github.RepositoryContent, files []*gitprovider.CommitFile, fileOpts gitprovider.FilesGetOptions) ([]*gitprovider.CommitFile, error) { + + // stop recursive calls when level is the max level reached + if fileOpts.MaxDepth <= 1 { + return nil, nil + } + + if !strings.HasSuffix(path, "/") { + path = path + "/" + } + subdirectoryPath := fmt.Sprintf("%s%s/", path, *dir.Name) + + // recursive call for child directories to get their content + childMaxFiles := fileOpts.MaxFiles - len(files) + childOptFns := gitprovider.FilesGetOptions{Recursive: fileOpts.Recursive, MaxFiles: childMaxFiles, MaxDepth: fileOpts.MaxDepth - 1} + childFiles, err := c.Get(ctx, subdirectoryPath, branch, &childOptFns) + + return childFiles, err + +} + // Get fetches and returns the contents of a file or directory from a given branch and path with possible options of FilesGetOption // If recursive option is provided, the files are retrieved recursively from subdirectories of the base path. // If recursive and MaxDepth options are provided, the files are retrieved recursively from subdirectories until reaching the max depth of levels @@ -76,27 +98,18 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git filePath := file.Path if *file.Type == "dir" { if fileOpts.Recursive { - // stop recursive calls when level is the max level reached - if fileOpts.MaxDepth <= 1 { - continue - } - - if !strings.HasSuffix(path, "/") { - path = path + "/" - } - subdirectoryPath := fmt.Sprintf("%s%s/", path, *file.Name) - - // recursive call for child directories to get their content - childMaxFiles := fileOpts.MaxFiles - len(files) - childOptFns := gitprovider.FilesGetOptions{Recursive: fileOpts.Recursive, MaxFiles: childMaxFiles, MaxDepth: fileOpts.MaxDepth - 1} - childFiles, err := c.Get(ctx, subdirectoryPath, branch, &childOptFns) + childFiles, err := c.getDirectoryFiles(ctx, path, branch, file, files, fileOpts) if err != nil { return nil, err } + if childFiles == nil { + continue + } + files = append(files, childFiles...) - files, limitSuccess := limitSliceSize(files, fileOpts.MaxFiles) - if limitSuccess { + files, maxFilesReached := limitSliceSize(files, fileOpts.MaxFiles) + if maxFilesReached { return files, nil } } diff --git a/github/integration_test.go b/github/integration_test.go index 5bc7eaec..326f2bff 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -635,6 +635,27 @@ var _ = Describe("GitHub Provider", func() { Expect(err).ToNot(HaveOccurred()) Expect(downloadedFiles).To(HaveLen(0)) + // Recursive option ,MaxDepth 3, all files check + // should be 3 + options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 3} + downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) + Expect(err).ToNot(HaveOccurred()) + Expect(downloadedFiles).To(HaveLen(3)) + + // Recursive option,MaxDepth 2,and MaxFiles 2 + // should be 2 + options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 2, MaxFiles: 2} + downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) + Expect(err).ToNot(HaveOccurred()) + Expect(downloadedFiles).To(HaveLen(2)) + + // Recursive option,MaxDepth 3,and MaxFiles 2 + // should be 2 + options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 3, MaxFiles: 2} + downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) + Expect(err).ToNot(HaveOccurred()) + Expect(downloadedFiles).To(HaveLen(2)) + }) AfterSuite(func() { From 2244841c69b07499d44c8539fb85f25f6e0999ad Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Sun, 17 Jul 2022 17:43:16 +0200 Subject: [PATCH 05/13] Remove recursive implementation from github file client get and its integration tests Signed-off-by: Rana Tarek Hassan --- github/client_repository_file.go | 61 -------------------- github/integration_test.go | 96 -------------------------------- gitprovider/options.go | 8 --- 3 files changed, 165 deletions(-) diff --git a/github/client_repository_file.go b/github/client_repository_file.go index 71f259c3..b01fffe8 100644 --- a/github/client_repository_file.go +++ b/github/client_repository_file.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "io/ioutil" - "strings" "github.com/fluxcd/go-git-providers/gitprovider" "github.com/google/go-github/v45/github" @@ -35,43 +34,7 @@ type FileClient struct { ref gitprovider.RepositoryRef } -// if number of files retrieved exceed the max number of files, return slice only until the number of maxFiles with bool true for limiting succeeded -func limitSliceSize(s []*gitprovider.CommitFile, size int) ([]*gitprovider.CommitFile, bool) { - if size != 0 && len(s) >= size { - return s[:size], true - } - return s, false -} - -// getDirectoryFiles will make the recursive call to get files in sub directory of a dir type -func (c *FileClient) getDirectoryFiles(ctx context.Context, path string, branch string, dir *github.RepositoryContent, files []*gitprovider.CommitFile, fileOpts gitprovider.FilesGetOptions) ([]*gitprovider.CommitFile, error) { - - // stop recursive calls when level is the max level reached - if fileOpts.MaxDepth <= 1 { - return nil, nil - } - - if !strings.HasSuffix(path, "/") { - path = path + "/" - } - subdirectoryPath := fmt.Sprintf("%s%s/", path, *dir.Name) - - // recursive call for child directories to get their content - childMaxFiles := fileOpts.MaxFiles - len(files) - childOptFns := gitprovider.FilesGetOptions{Recursive: fileOpts.Recursive, MaxFiles: childMaxFiles, MaxDepth: fileOpts.MaxDepth - 1} - childFiles, err := c.Get(ctx, subdirectoryPath, branch, &childOptFns) - - return childFiles, err - -} - // Get fetches and returns the contents of a file or directory from a given branch and path with possible options of FilesGetOption -// If recursive option is provided, the files are retrieved recursively from subdirectories of the base path. -// If recursive and MaxDepth options are provided, the files are retrieved recursively from subdirectories until reaching the max depth of levels -// Recursive and MaxDepth should be provided together, default MaxDepth : 0 -// If maxFiles option is provided, the number of files are maximized to the number provided - -// Uses https://docs.github.com/en/rest/repos/contents#get-repository-content func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) { opts := &github.RepositoryContentGetOptions{ @@ -96,26 +59,6 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git for _, file := range directoryContent { filePath := file.Path - if *file.Type == "dir" { - if fileOpts.Recursive { - childFiles, err := c.getDirectoryFiles(ctx, path, branch, file, files, fileOpts) - - if err != nil { - return nil, err - } - if childFiles == nil { - continue - } - - files = append(files, childFiles...) - files, maxFilesReached := limitSliceSize(files, fileOpts.MaxFiles) - if maxFilesReached { - return files, nil - } - } - continue - - } output, _, err := c.c.Client().Repositories.DownloadContents(ctx, c.ref.GetIdentity(), c.ref.GetRepository(), *filePath, opts) if err != nil { return nil, err @@ -133,10 +76,6 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git Path: filePath, Content: &contentStr, }) - files, limitSuccess := limitSliceSize(files, fileOpts.MaxFiles) - if limitSuccess { - return files, nil - } } diff --git a/github/integration_test.go b/github/integration_test.go index 326f2bff..05d8c599 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -562,102 +562,6 @@ var _ = Describe("GitHub Provider", func() { }) - It("should be possible to download files from path and branch specified with nested directory and max options", func() { - - userRepoRef := newUserRepoRef(testUser, testUserRepoName) - - userRepo, err := c.UserRepositories().Get(ctx, userRepoRef) - Expect(err).ToNot(HaveOccurred()) - - defaultBranch := userRepo.Get().DefaultBranch - - path0 := "clustersDir/cluster/machine.yaml" - content0 := "machine yaml content" - path1 := "clustersDir/cluster/machine1.yaml" - content1 := "machine1 yaml content" - path2 := "clustersDir/cluster2/clusterSubDir/machine2.yaml" - content2 := "machine2 yaml content" - - files := []gitprovider.CommitFile{ - { - Path: &path0, - Content: &content0, - }, - { - Path: &path1, - Content: &content1, - }, - { - Path: &path2, - Content: &content2, - }, - } - - commitFiles := make([]gitprovider.CommitFile, 0) - for _, file := range files { - path := file.Path - content := file.Content - commitFiles = append(commitFiles, gitprovider.CommitFile{ - Path: path, - Content: content, - }) - } - - _, err = userRepo.Commits().Create(ctx, *defaultBranch, "added config files", commitFiles) - Expect(err).ToNot(HaveOccurred()) - - //Recursive option check - options := gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true)} - downloadedFiles, err := userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) - Expect(err).ToNot(HaveOccurred()) - for ind, downloadedFile := range downloadedFiles { - Expect(*downloadedFile).To(Equal(files[ind])) - } - - //Max depth option check - //should be files in 2 levels: machine.yaml and machine1.yaml in level 2 (level 1 has cluster and cluster2 dirs) - options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 2} - downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) - Expect(err).ToNot(HaveOccurred()) - Expect(downloadedFiles).To(HaveLen(2)) - - // Max files option check with recursion - // should be 1 file :machine.yaml - options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 2, MaxFiles: 1} - downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) - Expect(err).ToNot(HaveOccurred()) - Expect(downloadedFiles).To(HaveLen(1)) - - // Max files option check with no recursion (base level) - // should be 0 - options = gitprovider.FilesGetOptions{MaxFiles: 2} - downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) - Expect(err).ToNot(HaveOccurred()) - Expect(downloadedFiles).To(HaveLen(0)) - - // Recursive option ,MaxDepth 3, all files check - // should be 3 - options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 3} - downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) - Expect(err).ToNot(HaveOccurred()) - Expect(downloadedFiles).To(HaveLen(3)) - - // Recursive option,MaxDepth 2,and MaxFiles 2 - // should be 2 - options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 2, MaxFiles: 2} - downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) - Expect(err).ToNot(HaveOccurred()) - Expect(downloadedFiles).To(HaveLen(2)) - - // Recursive option,MaxDepth 3,and MaxFiles 2 - // should be 2 - options = gitprovider.FilesGetOptions{Recursive: *gitprovider.BoolVar(true), MaxDepth: 3, MaxFiles: 2} - downloadedFiles, err = userRepo.Files().Get(ctx, "clustersDir", *defaultBranch, &options) - Expect(err).ToNot(HaveOccurred()) - Expect(downloadedFiles).To(HaveLen(2)) - - }) - AfterSuite(func() { if os.Getenv("SKIP_CLEANUP") == "1" { return diff --git a/gitprovider/options.go b/gitprovider/options.go index de9bd48a..07008d45 100644 --- a/gitprovider/options.go +++ b/gitprovider/options.go @@ -81,8 +81,6 @@ func (opts *RepositoryCreateOptions) ValidateOptions() error { // FilesGetOptions specifies optional options when fetcing files. type FilesGetOptions struct { Recursive bool - MaxFiles int - MaxDepth int } type FilesGetOption interface { @@ -94,10 +92,4 @@ func (opts *FilesGetOptions) ApplyFilesGetOptions(target *FilesGetOptions) { if opts.Recursive == true { target.Recursive = opts.Recursive } - if opts.MaxFiles > 0 { - target.MaxFiles = opts.MaxFiles - } - if opts.MaxDepth > 0 { - target.MaxDepth = opts.MaxDepth - } } From 40bc74d3ab64526bb92b21dc0665adef161a74f7 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Sun, 17 Jul 2022 19:41:37 +0200 Subject: [PATCH 06/13] If applied, this commit will add tree client to github gitprovider Add create,get, and list functionalities to tree client of github not implemented in gitlab and stash clients Signed-off-by: Rana Tarek Hassan --- github/client_repository_tree.go | 150 +++++++++++++++++++++++++++++++ github/integration_test.go | 81 +++++++++++++++++ github/resource_repository.go | 9 ++ gitlab/client_repository_tree.go | 50 +++++++++++ gitlab/resource_repository.go | 9 ++ gitprovider/client.go | 9 ++ gitprovider/resources.go | 16 +++- gitprovider/types_repository.go | 14 +++ stash/client_repository_tree.go | 50 +++++++++++ stash/resource_repository.go | 9 ++ 10 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 github/client_repository_tree.go create mode 100644 gitlab/client_repository_tree.go create mode 100644 stash/client_repository_tree.go diff --git a/github/client_repository_tree.go b/github/client_repository_tree.go new file mode 100644 index 00000000..0c206c1d --- /dev/null +++ b/github/client_repository_tree.go @@ -0,0 +1,150 @@ +/* +Copyright 2020 The Flux CD contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "context" + + "github.com/fluxcd/go-git-providers/gitprovider" + "github.com/google/go-github/v42/github" +) + +// TreeClient implements the gitprovider.TreeClient interface. +var _ gitprovider.TreeClient = &TreeClient{} + +// TreeClient operates on the trees in a specific repository. +type TreeClient struct { + *clientContext + ref gitprovider.RepositoryRef +} + +// Create creates,updates,deletes a tree +func (c *TreeClient) Create(ctx context.Context, tree *gitprovider.TreeInfo) (*gitprovider.TreeInfo, error) { + repoName := c.ref.GetRepository() + repoOwner := c.ref.GetIdentity() + + treeEntries := make([]*github.TreeEntry, 0) + for _, treeEntry := range tree.Tree { + treeEntries = append(treeEntries, &github.TreeEntry{ + Path: &treeEntry.Path, + Mode: &treeEntry.Mode, + Type: &treeEntry.Type, + Size: &treeEntry.Size, + SHA: &treeEntry.SHA, + URL: &treeEntry.URL, + }) + } + githubTree, _, err := c.c.Client().Git.CreateTree(ctx, repoOwner, repoName, tree.SHA, treeEntries) + if err != nil { + return nil, err + } + + responseTreeEntries := make([]*gitprovider.TreeEntry, 0) + for _, responseTreeEntry := range githubTree.Entries { + size := 0 + if *responseTreeEntry.Type != "tree" { + size = *responseTreeEntry.Size + } + responseTreeEntries = append(responseTreeEntries, &gitprovider.TreeEntry{ + Path: *responseTreeEntry.Path, + Mode: *responseTreeEntry.Mode, + Type: *responseTreeEntry.Type, + Size: size, + SHA: *responseTreeEntry.SHA, + URL: *responseTreeEntry.URL, + }) + } + + responseTreeInfo := gitprovider.TreeInfo{ + SHA: *githubTree.SHA, + Tree: responseTreeEntries, + Truncated: *githubTree.Truncated, + } + + return &responseTreeInfo, nil +} + +// Get returns a tree +func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) { + // GET /repos/{owner}/{repo}/git/trees + repoName := c.ref.GetRepository() + repoOwner := c.ref.GetIdentity() + githubTree, _, err := c.c.Client().Git.GetTree(ctx, repoOwner, repoName, sha, true) + if err != nil { + return nil, err + } + + treeEntries := make([]*gitprovider.TreeEntry, 0) + for _, treeEntry := range githubTree.Entries { + size := 0 + if *treeEntry.Type != "tree" { + size = *treeEntry.Size + } + treeEntries = append(treeEntries, &gitprovider.TreeEntry{ + Path: *treeEntry.Path, + Mode: *treeEntry.Mode, + Type: *treeEntry.Type, + Size: size, + SHA: *treeEntry.SHA, + URL: *treeEntry.URL, + }) + } + + treeInfo := gitprovider.TreeInfo{ + SHA: *githubTree.SHA, + Tree: treeEntries, + Truncated: *githubTree.Truncated, + } + + return &treeInfo, nil + +} + +// List files (blob) in a tree +func (c *TreeClient) List(ctx context.Context, sha string, recursive bool) ([]*gitprovider.TreeEntry, error) { + treeInfo, err := c.Get(ctx, sha, recursive) + if err != nil { + return nil, err + } + treeEntries := make([]*gitprovider.TreeEntry, 0) + for _, treeEntry := range treeInfo.Tree { + if treeEntry.Type == "blob" { + treeEntries = append(treeEntries, &gitprovider.TreeEntry{ + Path: treeEntry.Path, + Mode: treeEntry.Mode, + Type: treeEntry.Type, + Size: treeEntry.Size, + SHA: treeEntry.SHA, + URL: treeEntry.URL, + }) + } + } + + return treeEntries, nil +} + +func createTreeEntry(githubTreeEntry github.TreeEntry) *gitprovider.TreeEntry { + newTreeEntry := gitprovider.TreeEntry{ + Path: *githubTreeEntry.Path, + Mode: *githubTreeEntry.Mode, + Type: *githubTreeEntry.Type, + Size: *githubTreeEntry.Size, + SHA: *githubTreeEntry.SHA, + URL: *githubTreeEntry.URL, + } + return &newTreeEntry +} diff --git a/github/integration_test.go b/github/integration_test.go index 05d8c599..9bbc15e8 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -562,6 +562,87 @@ var _ = Describe("GitHub Provider", func() { }) + It("should be possible to get and list repo tree", func() { + + userRepoRef := newUserRepoRef(testUser, testUserRepoName) + + userRepo, err := c.UserRepositories().Get(ctx, userRepoRef) + Expect(err).ToNot(HaveOccurred()) + + defaultBranch := userRepo.Get().DefaultBranch + + path0 := "clustersDir/cluster/machine.yaml" + content0 := "machine yaml content" + path1 := "clustersDir/cluster/machine1.yaml" + content1 := "machine1 yaml content" + path2 := "clustersDir/cluster2/clusterSubDir/machine2.yaml" + content2 := "machine2 yaml content" + + files := []gitprovider.CommitFile{ + { + Path: &path0, + Content: &content0, + }, + { + Path: &path1, + Content: &content1, + }, + { + Path: &path2, + Content: &content2, + }, + } + + commitFiles := make([]gitprovider.CommitFile, 0) + for _, file := range files { + path := file.Path + content := file.Content + commitFiles = append(commitFiles, gitprovider.CommitFile{ + Path: path, + Content: content, + }) + } + + commit, err := userRepo.Commits().Create(ctx, *defaultBranch, "added files", commitFiles) + Expect(err).ToNot(HaveOccurred()) + commitSha := commit.Get().Sha + + // get tree + tree, err := userRepo.Trees().Get(ctx, commitSha, true) + Expect(err).ToNot(HaveOccurred()) + + // Tree should have length 9 for : LISENCE, README.md, 3 blob (files), 4 tree (directories) + Expect(tree.Tree).To(HaveLen(9)) + + // itemsToBeIgnored initially with 2 for LICENSE and README.md, and will also include tree types + itemsToBeIgnored := 2 + for ind, treeEntry := range tree.Tree { + if treeEntry.Type == "blob" { + if treeEntry.Path == "LICENSE" || treeEntry.Path == "README.md" { + continue + } + Expect(*&treeEntry.Path).To(Equal(*files[ind-itemsToBeIgnored].Path)) + continue + + } + itemsToBeIgnored += 1 + } + + // List tree items + treeEntries, err := userRepo.Trees().List(ctx, commitSha, true) + Expect(err).ToNot(HaveOccurred()) + + // Tree Entries should have length 5 for : LISENCE, README.md, 3 blob (files) + Expect(treeEntries).To(HaveLen(5)) + for ind, treeEntry := range treeEntries { + if treeEntry.Path == "LICENSE" || treeEntry.Path == "README.md" { + continue + } + Expect(*&treeEntry.Path).To(Equal(*files[ind-2].Path)) + } + + }) + AfterSuite(func() { if os.Getenv("SKIP_CLEANUP") == "1" { return diff --git a/github/resource_repository.go b/github/resource_repository.go index 13fda824..abfcf886 100644 --- a/github/resource_repository.go +++ b/github/resource_repository.go @@ -78,6 +78,10 @@ func newUserRepository(ctx *clientContext, apiObj *github.Repository, ref gitpro clientContext: ctx, ref: ref, }, + trees: &TreeClient{ + clientContext: ctx, + ref: ref, + }, } } @@ -95,6 +99,7 @@ type userRepository struct { branches *BranchClient pullRequests *PullRequestClient files *FileClient + trees *TreeClient } func (r *userRepository) Get() gitprovider.RepositoryInfo { @@ -137,6 +142,10 @@ func (r *userRepository) Files() gitprovider.FileClient { return r.files } +func (r *userRepository) Trees() gitprovider.TreeClient { + return r.trees +} + // Update will apply the desired state in this object to the server. // Only set fields will be respected (i.e. PATCH behaviour). // In order to apply changes to this object, use the .Set({Resource}Info) error diff --git a/gitlab/client_repository_tree.go b/gitlab/client_repository_tree.go new file mode 100644 index 00000000..7dbfd391 --- /dev/null +++ b/gitlab/client_repository_tree.go @@ -0,0 +1,50 @@ +/* +Copyright 2020 The Flux CD contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gitlab + +import ( + "context" + "fmt" + + "github.com/fluxcd/go-git-providers/gitprovider" +) + +// TreeClient implements the gitprovider.TreeClient interface. +var _ gitprovider.TreeClient = &TreeClient{} + +// TreeClient operates on the trees in a specific repository. +type TreeClient struct { + *clientContext + ref gitprovider.RepositoryRef +} + +// Create creates,updates,deletes a tree +func (c *TreeClient) Create(ctx context.Context, tree *gitprovider.TreeInfo) (*gitprovider.TreeInfo, error) { + return nil, fmt.Errorf("error creaing tree %s. not implemented in gitlab yet", tree.SHA) + +} + +// Get returns a tree +func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) { + return nil, fmt.Errorf("error getting tree %s. not implemented in gitlab yet", sha) + +} + +// List files (blob) in a tree +func (c *TreeClient) List(ctx context.Context, sha string, recursive bool) ([]*gitprovider.TreeEntry, error) { + return nil, fmt.Errorf("error listing tree items %s. not implemented in gitlab yet", sha) +} diff --git a/gitlab/resource_repository.go b/gitlab/resource_repository.go index eae17395..36468f6a 100644 --- a/gitlab/resource_repository.go +++ b/gitlab/resource_repository.go @@ -51,6 +51,10 @@ func newUserProject(ctx *clientContext, apiObj *gogitlab.Project, ref gitprovide clientContext: ctx, ref: ref, }, + trees: &TreeClient{ + clientContext: ctx, + ref: ref, + }, } } @@ -67,6 +71,7 @@ type userProject struct { branches *BranchClient pullRequests *PullRequestClient files *FileClient + trees *TreeClient } func (p *userProject) Get() gitprovider.RepositoryInfo { @@ -109,6 +114,10 @@ func (p *userProject) Files() gitprovider.FileClient { return p.files } +func (p *userProject) Trees() gitprovider.TreeClient { + return p.trees +} + // The internal API object will be overridden with the received server data. func (p *userProject) Update(ctx context.Context) error { // PATCH /repos/{owner}/{repo} diff --git a/gitprovider/client.go b/gitprovider/client.go index 55726b8e..3be0c428 100644 --- a/gitprovider/client.go +++ b/gitprovider/client.go @@ -243,3 +243,12 @@ type FileClient interface { // GetFiles fetch files content from specific path and branch Get(ctx context.Context, path, branch string, optFns ...FilesGetOption) ([]*CommitFile, error) } + +type TreeClient interface { + // Create allows for creating or editing tree + Create(ctx context.Context, tree *TreeInfo) (*TreeInfo, error) + // Get retrieves tree information and items + Get(ctx context.Context, sha string, recursive bool) (*TreeInfo, error) + // List retrieves list of tree files (files/blob) + List(ctx context.Context, sha string, recursive bool) ([]*TreeEntry, error) +} diff --git a/gitprovider/resources.go b/gitprovider/resources.go index 257e109b..65f61020 100644 --- a/gitprovider/resources.go +++ b/gitprovider/resources.go @@ -77,8 +77,11 @@ type UserRepository interface { // PullRequests gives access to this specific repository pull requests PullRequests() PullRequestClient - // Files gives access to this specific repository pull requests + // Files gives access to this specific repository files Files() FileClient + + // Trees gives access to this specific repository trees + Trees() TreeClient } // OrgRepository describes a repository owned by an organization. @@ -156,3 +159,14 @@ type PullRequest interface { // Get returns high-level information about this pull request. Get() PullRequestInfo } + +type Tree interface { + // Object implements the Object interface, + // allowing access to the underlying object returned from the API. + Object + + // Get returns high-level information about this tree. + Create() TreeInfo + Get() TreeInfo + List() TreeEntry +} diff --git a/gitprovider/types_repository.go b/gitprovider/types_repository.go index 7c815c44..f8af74f2 100644 --- a/gitprovider/types_repository.go +++ b/gitprovider/types_repository.go @@ -225,3 +225,17 @@ type PullRequestInfo struct { // +required WebURL string `json:"web_url"` } + +type TreeEntry struct { + Path string `json:"path"` + Mode string `json:"mode"` + Type string `json:"type"` + Size int `json:"size"` + SHA string `json:"sha"` + URL string `json:"url"` +} +type TreeInfo struct { + SHA string `json:"sha"` + Tree []*TreeEntry `json:"tree"` + Truncated bool `json:"truncated"` +} diff --git a/stash/client_repository_tree.go b/stash/client_repository_tree.go new file mode 100644 index 00000000..2a8be8f6 --- /dev/null +++ b/stash/client_repository_tree.go @@ -0,0 +1,50 @@ +/* +Copyright 2020 The Flux CD contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package stash + +import ( + "context" + "fmt" + + "github.com/fluxcd/go-git-providers/gitprovider" +) + +// TreeClient implements the gitprovider.TreeClient interface. +var _ gitprovider.TreeClient = &TreeClient{} + +// TreeClient operates on the trees in a specific repository. +type TreeClient struct { + *clientContext + ref gitprovider.RepositoryRef +} + +// Create creates,updates,deletes a tree +func (c *TreeClient) Create(ctx context.Context, tree *gitprovider.TreeInfo) (*gitprovider.TreeInfo, error) { + return nil, fmt.Errorf("error creaing tree %s. not implemented in stash yet", tree.SHA) + +} + +// Get returns a tree +func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) { + return nil, fmt.Errorf("error getting tree %s. not implemented in stash yet", sha) + +} + +// List files (blob) in a tree +func (c *TreeClient) List(ctx context.Context, sha string, recursive bool) ([]*gitprovider.TreeEntry, error) { + return nil, fmt.Errorf("error listing tree items %s. not implemented in stash yet", sha) +} diff --git a/stash/resource_repository.go b/stash/resource_repository.go index 7dda26ce..66afd0ac 100644 --- a/stash/resource_repository.go +++ b/stash/resource_repository.go @@ -52,6 +52,10 @@ func newUserRepository(ctx *clientContext, apiObj *Repository, ref gitprovider.R clientContext: ctx, ref: ref, }, + trees: &TreeClient{ + clientContext: ctx, + ref: ref, + }, } } @@ -66,6 +70,7 @@ type userRepository struct { pullRequests *PullRequestClient commits *CommitClient files *FileClient + trees *TreeClient } func (r *userRepository) Branches() gitprovider.BranchClient { @@ -84,6 +89,10 @@ func (r *userRepository) Files() gitprovider.FileClient { return r.files } +func (r *userRepository) Trees() gitprovider.TreeClient { + return r.trees +} + func (r *userRepository) Get() gitprovider.RepositoryInfo { return repositoryFromAPI(&r.repository) } From bfd6fc7e816e533281e710175303b30eb4f2cdb5 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Mon, 18 Jul 2022 17:25:33 +0200 Subject: [PATCH 07/13] Apply suggestions from code review LICENSE typo Co-authored-by: Kevin McDermott Signed-off-by: Rana Tarek Hassan --- github/integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github/integration_test.go b/github/integration_test.go index 9bbc15e8..dfc627c5 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -611,7 +611,7 @@ var _ = Describe("GitHub Provider", func() { tree, err := userRepo.Trees().Get(ctx, commitSha, true) Expect(err).ToNot(HaveOccurred()) - // Tree should have length 9 for : LISENCE, README.md, 3 blob (files), 4 tree (directories) + // Tree should have length 9 for : LICENSE, README.md, 3 blob (files), 4 tree (directories) Expect(tree.Tree).To(HaveLen(9)) // itemsToBeIgnored initially with 2 for LICENSE and README.md, and will also include tree types @@ -632,7 +632,7 @@ var _ = Describe("GitHub Provider", func() { treeEntries, err := userRepo.Trees().List(ctx, commitSha, true) Expect(err).ToNot(HaveOccurred()) - // Tree Entries should have length 5 for : LISENCE, README.md, 3 blob (files) + // Tree Entries should have length 5 for : LICENSE, README.md, 3 blob (files) Expect(treeEntries).To(HaveLen(5)) for ind, treeEntry := range treeEntries { if treeEntry.Path == "LICENSE" || treeEntry.Path == "README.md" { From 4fd5ddaf692c3b03c22155a44d87db737d97d215 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Tue, 19 Jul 2022 15:53:33 +0200 Subject: [PATCH 08/13] If applied, this commit will add some GoDocs and apply suggestions from code review Signed-off-by: Rana Tarek Hassan --- github/client_repository_tree.go | 24 ++++++------------------ github/integration_test.go | 4 ++-- gitprovider/client.go | 2 ++ gitprovider/options.go | 7 ++++--- gitprovider/resources.go | 1 + gitprovider/types_repository.go | 26 ++++++++++++++++++++------ 6 files changed, 35 insertions(+), 29 deletions(-) diff --git a/github/client_repository_tree.go b/github/client_repository_tree.go index 0c206c1d..9f6c637d 100644 --- a/github/client_repository_tree.go +++ b/github/client_repository_tree.go @@ -78,30 +78,30 @@ func (c *TreeClient) Create(ctx context.Context, tree *gitprovider.TreeInfo) (*g return &responseTreeInfo, nil } -// Get returns a tree +// Get returns a single tree using the SHA1 value for that tree. func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) { // GET /repos/{owner}/{repo}/git/trees repoName := c.ref.GetRepository() repoOwner := c.ref.GetIdentity() - githubTree, _, err := c.c.Client().Git.GetTree(ctx, repoOwner, repoName, sha, true) + githubTree, _, err := c.c.Client().Git.GetTree(ctx, repoOwner, repoName, sha, recursive) if err != nil { return nil, err } - treeEntries := make([]*gitprovider.TreeEntry, 0) - for _, treeEntry := range githubTree.Entries { + treeEntries := make([]*gitprovider.TreeEntry, len(githubTree.Entries)) + for ind, treeEntry := range githubTree.Entries { size := 0 if *treeEntry.Type != "tree" { size = *treeEntry.Size } - treeEntries = append(treeEntries, &gitprovider.TreeEntry{ + treeEntries[ind] = &gitprovider.TreeEntry{ Path: *treeEntry.Path, Mode: *treeEntry.Mode, Type: *treeEntry.Type, Size: size, SHA: *treeEntry.SHA, URL: *treeEntry.URL, - }) + } } treeInfo := gitprovider.TreeInfo{ @@ -136,15 +136,3 @@ func (c *TreeClient) List(ctx context.Context, sha string, recursive bool) ([]*g return treeEntries, nil } - -func createTreeEntry(githubTreeEntry github.TreeEntry) *gitprovider.TreeEntry { - newTreeEntry := gitprovider.TreeEntry{ - Path: *githubTreeEntry.Path, - Mode: *githubTreeEntry.Mode, - Type: *githubTreeEntry.Type, - Size: *githubTreeEntry.Size, - SHA: *githubTreeEntry.SHA, - URL: *githubTreeEntry.URL, - } - return &newTreeEntry -} diff --git a/github/integration_test.go b/github/integration_test.go index dfc627c5..4d6fac9d 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -621,7 +621,7 @@ var _ = Describe("GitHub Provider", func() { if treeEntry.Path == "LICENSE" || treeEntry.Path == "README.md" { continue } - Expect(*&treeEntry.Path).To(Equal(*files[ind-itemsToBeIgnored].Path)) + Expect(treeEntry.Path).To(Equal(*files[ind-itemsToBeIgnored].Path)) continue } @@ -638,7 +638,7 @@ var _ = Describe("GitHub Provider", func() { if treeEntry.Path == "LICENSE" || treeEntry.Path == "README.md" { continue } - Expect(*&treeEntry.Path).To(Equal(*files[ind-2].Path)) + Expect(treeEntry.Path).To(Equal(*files[ind-2].Path)) } }) diff --git a/gitprovider/client.go b/gitprovider/client.go index 3be0c428..1d2da343 100644 --- a/gitprovider/client.go +++ b/gitprovider/client.go @@ -244,6 +244,8 @@ type FileClient interface { Get(ctx context.Context, path, branch string, optFns ...FilesGetOption) ([]*CommitFile, error) } +// TreeClient operates on the trees for a Git repository which describe the hierarchy between files in the repository +// This client can be accessed through Repository.Trees() type TreeClient interface { // Create allows for creating or editing tree Create(ctx context.Context, tree *TreeInfo) (*TreeInfo, error) diff --git a/gitprovider/options.go b/gitprovider/options.go index 07008d45..a3c30ba9 100644 --- a/gitprovider/options.go +++ b/gitprovider/options.go @@ -83,13 +83,14 @@ type FilesGetOptions struct { Recursive bool } +// FilesGetOption is an interface for applying options when fetching/getting files type FilesGetOption interface { ApplyFilesGetOptions(target *FilesGetOptions) } +// ApplyFilesGetOptions applies target options onto the invoked opts func (opts *FilesGetOptions) ApplyFilesGetOptions(target *FilesGetOptions) { // Go through each field in opts, and apply it to target if set - if opts.Recursive == true { - target.Recursive = opts.Recursive - } + target.Recursive = opts.Recursive + } diff --git a/gitprovider/resources.go b/gitprovider/resources.go index 65f61020..2450bd75 100644 --- a/gitprovider/resources.go +++ b/gitprovider/resources.go @@ -160,6 +160,7 @@ type PullRequest interface { Get() PullRequestInfo } +// Tree represents a git tree which represents the hierarchy between files in a Git repository type Tree interface { // Object implements the Object interface, // allowing access to the underlying object returned from the API. diff --git a/gitprovider/types_repository.go b/gitprovider/types_repository.go index f8af74f2..27f3afe1 100644 --- a/gitprovider/types_repository.go +++ b/gitprovider/types_repository.go @@ -226,16 +226,30 @@ type PullRequestInfo struct { WebURL string `json:"web_url"` } +// TreeEntry contains info about each tree object's structure in TreeInfo whether it is a file or tree type TreeEntry struct { + // Path is the path of the file/blob or sub tree in a tree Path string `json:"path"` + // Mode of the file/tree. + // (100644:file (blob), 100755:executable (blob), 040000:subdirectory(tree),160000:submodule(commit),120000:blob that specifies the path of a symlink) Mode string `json:"mode"` + // Type is the item type, It is either blob, tree, or commit. Type string `json:"type"` - Size int `json:"size"` - SHA string `json:"sha"` - URL string `json:"url"` + // Size is the size of the file/blob if the type is a blob, it is not populated if the type is a tree + Size int `json:"size"` + // SHA is the SHA1 checksum ID of the object in the tree + SHA string `json:"sha"` + // URL is the url that can be used to retrieve the details of the blob, tree of commit + URL string `json:"url"` } + +// TreeInfo contains high-level information about a git Tree representing the hierarchy between files in a Git repository type TreeInfo struct { - SHA string `json:"sha"` - Tree []*TreeEntry `json:"tree"` - Truncated bool `json:"truncated"` + // SHA is the SHA1 checksum ID of the tree, or the branch name + SHA string `json:"sha"` + // Tree is the list of TreeEntry objects describing the structure of the tree + Tree []*TreeEntry `json:"tree"` + // Truncated represents whether a tree is truncated when fetching a tree + // If truncated is true in the response when fetching a tree, then the number of items in the tree array exceeded the maximum limit + Truncated bool `json:"truncated"` } From cbb43045fc38e4647d63c5393810b33595a2f992 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Sun, 24 Jul 2022 13:56:25 +0200 Subject: [PATCH 09/13] Apply suggestions from code review Co-authored-by: souleb Signed-off-by: Rana Tarek Hassan --- gitprovider/resources.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitprovider/resources.go b/gitprovider/resources.go index 2450bd75..1b48c615 100644 --- a/gitprovider/resources.go +++ b/gitprovider/resources.go @@ -80,7 +80,7 @@ type UserRepository interface { // Files gives access to this specific repository files Files() FileClient - // Trees gives access to this specific repository trees + // Trees gives access to this specific repository trees. Trees() TreeClient } @@ -160,7 +160,7 @@ type PullRequest interface { Get() PullRequestInfo } -// Tree represents a git tree which represents the hierarchy between files in a Git repository +// Tree represents a git tree which is the hierarchical structure of your git data. type Tree interface { // Object implements the Object interface, // allowing access to the underlying object returned from the API. From c07cba13eeb4eb58eef48f1e8b7fa1b0ee016184 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Sun, 24 Jul 2022 22:05:26 +0200 Subject: [PATCH 10/13] If applied, this commit will add defer for close of Files Get It will add go docs based on suggestions from code review Signed-off-by: Rana Tarek Hassan --- github/client_repository_file.go | 10 +++++----- github/client_repository_tree.go | 1 + gitlab/client_repository_file.go | 4 +++- gitprovider/resources.go | 4 +++- gitprovider/types_repository.go | 2 ++ 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/github/client_repository_file.go b/github/client_repository_file.go index b01fffe8..6030b925 100644 --- a/github/client_repository_file.go +++ b/github/client_repository_file.go @@ -34,7 +34,9 @@ type FileClient struct { ref gitprovider.RepositoryRef } -// Get fetches and returns the contents of a file or directory from a given branch and path with possible options of FilesGetOption +// Get fetches and returns the contents of a file or multiple files in a directory from a given branch and path with possible options of FilesGetOption +// If a file path is given, the contents of the file are returned +// If a directory path is given, the contents of the files in the path's root are returned func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) { opts := &github.RepositoryContentGetOptions{ @@ -67,10 +69,8 @@ func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...git if err != nil { return nil, err } - err = output.Close() - if err != nil { - return nil, err - } + defer output.Close() + contentStr := string(content) files = append(files, &gitprovider.CommitFile{ Path: filePath, diff --git a/github/client_repository_tree.go b/github/client_repository_tree.go index 9f6c637d..7007f43a 100644 --- a/github/client_repository_tree.go +++ b/github/client_repository_tree.go @@ -79,6 +79,7 @@ func (c *TreeClient) Create(ctx context.Context, tree *gitprovider.TreeInfo) (*g } // Get returns a single tree using the SHA1 value for that tree. +// uses https://docs.github.com/en/rest/git/trees#get-a-tree func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) { // GET /repos/{owner}/{repo}/git/trees repoName := c.ref.GetRepository() diff --git a/gitlab/client_repository_file.go b/gitlab/client_repository_file.go index 73dd57c9..cfcb2ef2 100644 --- a/gitlab/client_repository_file.go +++ b/gitlab/client_repository_file.go @@ -35,7 +35,9 @@ type FileClient struct { ref gitprovider.RepositoryRef } -// Get fetches and returns the contents of a file from a given branch and path +// Get fetches and returns the contents of a file or multiple files in a directory from a given branch and path with possible options of FilesGetOption +// If a file path is given, the contents of the file are returned +// If a directory path is given, the contents of the files in the path's root are returned func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) { filesGetOpts := gitprovider.FilesGetOptions{} diff --git a/gitprovider/resources.go b/gitprovider/resources.go index 1b48c615..04a6a303 100644 --- a/gitprovider/resources.go +++ b/gitprovider/resources.go @@ -166,8 +166,10 @@ type Tree interface { // allowing access to the underlying object returned from the API. Object - // Get returns high-level information about this tree. + // Create allows for creating or editing tree Create() TreeInfo + // Get returns high-level information about this tree. Get() TreeInfo + // List files (blob) in a tree List() TreeEntry } diff --git a/gitprovider/types_repository.go b/gitprovider/types_repository.go index 27f3afe1..f05d75e0 100644 --- a/gitprovider/types_repository.go +++ b/gitprovider/types_repository.go @@ -239,6 +239,8 @@ type TreeEntry struct { Size int `json:"size"` // SHA is the SHA1 checksum ID of the object in the tree SHA string `json:"sha"` + // Content is the content of a blob file, either content aor sha are set. If both are using Github will return an error + Content string `json:"content"` // URL is the url that can be used to retrieve the details of the blob, tree of commit URL string `json:"url"` } From 1c4bef292f3cb802ef0f05bfd43ad014c4c14396 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Tue, 9 Aug 2022 23:06:55 +0200 Subject: [PATCH 11/13] If applied, this commit will add implementation for List Tree items in gitlab client It will add path parameter used in github tree list to optionally filter files in a certain path Signed-off-by: Rana Tarek Hassan --- github/client_repository_tree.go | 23 ++++++++++-------- github/integration_test.go | 14 +++++++++-- gitlab/client_repository_tree.go | 32 ++++++++++++++++++++++--- gitlab/integration_test.go | 41 ++++++++++++++++++++++++++++++++ gitprovider/client.go | 4 ++-- gitprovider/types_repository.go | 2 ++ stash/client_repository_tree.go | 2 +- 7 files changed, 100 insertions(+), 18 deletions(-) diff --git a/github/client_repository_tree.go b/github/client_repository_tree.go index 7007f43a..d632e625 100644 --- a/github/client_repository_tree.go +++ b/github/client_repository_tree.go @@ -18,6 +18,7 @@ package github import ( "context" + "strings" "github.com/fluxcd/go-git-providers/gitprovider" "github.com/google/go-github/v42/github" @@ -115,8 +116,8 @@ func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitp } -// List files (blob) in a tree -func (c *TreeClient) List(ctx context.Context, sha string, recursive bool) ([]*gitprovider.TreeEntry, error) { +// List files (blob) in a tree givent the tree sha (path is not used with Github Tree client) +func (c *TreeClient) List(ctx context.Context, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) { treeInfo, err := c.Get(ctx, sha, recursive) if err != nil { return nil, err @@ -124,14 +125,16 @@ func (c *TreeClient) List(ctx context.Context, sha string, recursive bool) ([]*g treeEntries := make([]*gitprovider.TreeEntry, 0) for _, treeEntry := range treeInfo.Tree { if treeEntry.Type == "blob" { - treeEntries = append(treeEntries, &gitprovider.TreeEntry{ - Path: treeEntry.Path, - Mode: treeEntry.Mode, - Type: treeEntry.Type, - Size: treeEntry.Size, - SHA: treeEntry.SHA, - URL: treeEntry.URL, - }) + if path == "" || (path != "" && strings.HasPrefix(treeEntry.Path, path)) { + treeEntries = append(treeEntries, &gitprovider.TreeEntry{ + Path: treeEntry.Path, + Mode: treeEntry.Mode, + Type: treeEntry.Type, + Size: treeEntry.Size, + SHA: treeEntry.SHA, + URL: treeEntry.URL, + }) + } } } diff --git a/github/integration_test.go b/github/integration_test.go index 4d6fac9d..d817161b 100644 --- a/github/integration_test.go +++ b/github/integration_test.go @@ -628,8 +628,8 @@ var _ = Describe("GitHub Provider", func() { itemsToBeIgnored += 1 } - // List tree items - treeEntries, err := userRepo.Trees().List(ctx, commitSha, true) + // List tree items with no path provided + treeEntries, err := userRepo.Trees().List(ctx, commitSha, "", true) Expect(err).ToNot(HaveOccurred()) // Tree Entries should have length 5 for : LICENSE, README.md, 3 blob (files) @@ -641,6 +641,16 @@ var _ = Describe("GitHub Provider", func() { Expect(treeEntry.Path).To(Equal(*files[ind-2].Path)) } + //List tree items with path provided to filter on + treeEntries, err = userRepo.Trees().List(ctx, commitSha, "clustersDir/", true) + Expect(err).ToNot(HaveOccurred()) + + // Tree Entries should have length 3 for :3 blob (files) + Expect(treeEntries).To(HaveLen(3)) + for ind, treeEntry := range treeEntries { + Expect(treeEntry.Path).To(Equal(*files[ind].Path)) + } + }) AfterSuite(func() { diff --git a/gitlab/client_repository_tree.go b/gitlab/client_repository_tree.go index 7dbfd391..80ea9788 100644 --- a/gitlab/client_repository_tree.go +++ b/gitlab/client_repository_tree.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/fluxcd/go-git-providers/gitprovider" + "github.com/xanzy/go-gitlab" ) // TreeClient implements the gitprovider.TreeClient interface. @@ -44,7 +45,32 @@ func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitp } -// List files (blob) in a tree -func (c *TreeClient) List(ctx context.Context, sha string, recursive bool) ([]*gitprovider.TreeEntry, error) { - return nil, fmt.Errorf("error listing tree items %s. not implemented in gitlab yet", sha) +// List files (blob) in a tree, sha is represented by the branch name +func (c *TreeClient) List(ctx context.Context, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) { + opts := &gitlab.ListTreeOptions{ + Path: &path, + Ref: &sha, + Recursive: &recursive, + } + + treeFiles, _, err := c.c.Client().Repositories.ListTree(getRepoPath(c.ref), opts) + if err != nil { + return nil, err + } + + treeEntries := make([]*gitprovider.TreeEntry, 0) + for _, treeEntry := range treeFiles { + if treeEntry.Type == "blob" { + size := 0 + treeEntries = append(treeEntries, &gitprovider.TreeEntry{ + Path: treeEntry.Path, + Mode: treeEntry.Mode, + Type: treeEntry.Type, + Size: size, + ID: treeEntry.ID, + }) + } + } + + return treeEntries, nil } diff --git a/gitlab/integration_test.go b/gitlab/integration_test.go index baaf9224..dc9e4294 100644 --- a/gitlab/integration_test.go +++ b/gitlab/integration_test.go @@ -937,6 +937,47 @@ var _ = Describe("GitLab Provider", func() { Expect(*downloadedFile).To(Equal(files[ind])) } + }) + It("should be possible list repo tree files", func() { + userRepoRef := newUserRepoRef(testUserName, testRepoName) + + userRepo, err := c.UserRepositories().Get(ctx, userRepoRef) + Expect(err).ToNot(HaveOccurred()) + + defaultBranch := userRepo.Get().DefaultBranch + + path0 := "clustersDir/cluster/machine.yaml" + content0 := "machine0 yaml content" + path1 := "clustersDir/cluster/machine1.yaml" + content1 := "machine1 yaml content" + path2 := "clustersDir/cluster2/clusterSubDir/machine2.yaml" + content2 := "machine2 yaml content" + + files := []gitprovider.CommitFile{ + { + Path: &path0, + Content: &content0, + }, + { + Path: &path1, + Content: &content1, + }, + { + Path: &path2, + Content: &content2, + }, + } + + // List tree items + treeEntries, err := userRepo.Trees().List(ctx, *defaultBranch, "clustersDir/", true) + Expect(err).ToNot(HaveOccurred()) + + // Tree Entries should have length 3 for : 3 blob (files) + Expect(treeEntries).To(HaveLen(3)) + for ind, treeEntry := range treeEntries { + Expect(treeEntry.Path).To(Equal(*files[ind].Path)) + } + }) AfterSuite(func() { diff --git a/gitprovider/client.go b/gitprovider/client.go index 1d2da343..86b4880b 100644 --- a/gitprovider/client.go +++ b/gitprovider/client.go @@ -251,6 +251,6 @@ type TreeClient interface { Create(ctx context.Context, tree *TreeInfo) (*TreeInfo, error) // Get retrieves tree information and items Get(ctx context.Context, sha string, recursive bool) (*TreeInfo, error) - // List retrieves list of tree files (files/blob) - List(ctx context.Context, sha string, recursive bool) ([]*TreeEntry, error) + // List retrieves list of tree files (files/blob) from given tree sha/id or path+branch + List(ctx context.Context, sha string, path string, recursive bool) ([]*TreeEntry, error) } diff --git a/gitprovider/types_repository.go b/gitprovider/types_repository.go index f05d75e0..24cc6ce6 100644 --- a/gitprovider/types_repository.go +++ b/gitprovider/types_repository.go @@ -243,6 +243,8 @@ type TreeEntry struct { Content string `json:"content"` // URL is the url that can be used to retrieve the details of the blob, tree of commit URL string `json:"url"` + // Id is the id of the tree entry retrieved from Gitlab (Optional) + ID string `json:"id"` } // TreeInfo contains high-level information about a git Tree representing the hierarchy between files in a Git repository diff --git a/stash/client_repository_tree.go b/stash/client_repository_tree.go index 2a8be8f6..86edeecc 100644 --- a/stash/client_repository_tree.go +++ b/stash/client_repository_tree.go @@ -45,6 +45,6 @@ func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitp } // List files (blob) in a tree -func (c *TreeClient) List(ctx context.Context, sha string, recursive bool) ([]*gitprovider.TreeEntry, error) { +func (c *TreeClient) List(ctx context.Context, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) { return nil, fmt.Errorf("error listing tree items %s. not implemented in stash yet", sha) } From 0a7b386c5a1ce735edffc5923c53ceb5dc9fb38c Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Mon, 15 Aug 2022 17:52:35 +0200 Subject: [PATCH 12/13] Upgrade usage of go-github to v45 in client_repository_tree Signed-off-by: Rana Tarek Hassan --- github/client_repository_tree.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github/client_repository_tree.go b/github/client_repository_tree.go index d632e625..09778390 100644 --- a/github/client_repository_tree.go +++ b/github/client_repository_tree.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/fluxcd/go-git-providers/gitprovider" - "github.com/google/go-github/v42/github" + "github.com/google/go-github/v45/github" ) // TreeClient implements the gitprovider.TreeClient interface. From 3b287c11d870ba22d5f4d1aca52cef2c96b2ba60 Mon Sep 17 00:00:00 2001 From: Rana Tarek Hassan Date: Wed, 17 Aug 2022 13:38:42 +0200 Subject: [PATCH 13/13] If applied, this will remove create from tree client interface Signed-off-by: Rana Tarek Hassan --- github/client_repository_tree.go | 47 -------------------------------- gitlab/client_repository_tree.go | 6 ---- gitprovider/client.go | 2 -- gitprovider/resources.go | 2 -- stash/client_repository_tree.go | 6 ---- 5 files changed, 63 deletions(-) diff --git a/github/client_repository_tree.go b/github/client_repository_tree.go index 09778390..65e9ac31 100644 --- a/github/client_repository_tree.go +++ b/github/client_repository_tree.go @@ -21,7 +21,6 @@ import ( "strings" "github.com/fluxcd/go-git-providers/gitprovider" - "github.com/google/go-github/v45/github" ) // TreeClient implements the gitprovider.TreeClient interface. @@ -33,52 +32,6 @@ type TreeClient struct { ref gitprovider.RepositoryRef } -// Create creates,updates,deletes a tree -func (c *TreeClient) Create(ctx context.Context, tree *gitprovider.TreeInfo) (*gitprovider.TreeInfo, error) { - repoName := c.ref.GetRepository() - repoOwner := c.ref.GetIdentity() - - treeEntries := make([]*github.TreeEntry, 0) - for _, treeEntry := range tree.Tree { - treeEntries = append(treeEntries, &github.TreeEntry{ - Path: &treeEntry.Path, - Mode: &treeEntry.Mode, - Type: &treeEntry.Type, - Size: &treeEntry.Size, - SHA: &treeEntry.SHA, - URL: &treeEntry.URL, - }) - } - githubTree, _, err := c.c.Client().Git.CreateTree(ctx, repoOwner, repoName, tree.SHA, treeEntries) - if err != nil { - return nil, err - } - - responseTreeEntries := make([]*gitprovider.TreeEntry, 0) - for _, responseTreeEntry := range githubTree.Entries { - size := 0 - if *responseTreeEntry.Type != "tree" { - size = *responseTreeEntry.Size - } - responseTreeEntries = append(responseTreeEntries, &gitprovider.TreeEntry{ - Path: *responseTreeEntry.Path, - Mode: *responseTreeEntry.Mode, - Type: *responseTreeEntry.Type, - Size: size, - SHA: *responseTreeEntry.SHA, - URL: *responseTreeEntry.URL, - }) - } - - responseTreeInfo := gitprovider.TreeInfo{ - SHA: *githubTree.SHA, - Tree: responseTreeEntries, - Truncated: *githubTree.Truncated, - } - - return &responseTreeInfo, nil -} - // Get returns a single tree using the SHA1 value for that tree. // uses https://docs.github.com/en/rest/git/trees#get-a-tree func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) { diff --git a/gitlab/client_repository_tree.go b/gitlab/client_repository_tree.go index 80ea9788..86e0ea4f 100644 --- a/gitlab/client_repository_tree.go +++ b/gitlab/client_repository_tree.go @@ -33,12 +33,6 @@ type TreeClient struct { ref gitprovider.RepositoryRef } -// Create creates,updates,deletes a tree -func (c *TreeClient) Create(ctx context.Context, tree *gitprovider.TreeInfo) (*gitprovider.TreeInfo, error) { - return nil, fmt.Errorf("error creaing tree %s. not implemented in gitlab yet", tree.SHA) - -} - // Get returns a tree func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) { return nil, fmt.Errorf("error getting tree %s. not implemented in gitlab yet", sha) diff --git a/gitprovider/client.go b/gitprovider/client.go index 86b4880b..63bd0c13 100644 --- a/gitprovider/client.go +++ b/gitprovider/client.go @@ -247,8 +247,6 @@ type FileClient interface { // TreeClient operates on the trees for a Git repository which describe the hierarchy between files in the repository // This client can be accessed through Repository.Trees() type TreeClient interface { - // Create allows for creating or editing tree - Create(ctx context.Context, tree *TreeInfo) (*TreeInfo, error) // Get retrieves tree information and items Get(ctx context.Context, sha string, recursive bool) (*TreeInfo, error) // List retrieves list of tree files (files/blob) from given tree sha/id or path+branch diff --git a/gitprovider/resources.go b/gitprovider/resources.go index 04a6a303..a8586745 100644 --- a/gitprovider/resources.go +++ b/gitprovider/resources.go @@ -166,8 +166,6 @@ type Tree interface { // allowing access to the underlying object returned from the API. Object - // Create allows for creating or editing tree - Create() TreeInfo // Get returns high-level information about this tree. Get() TreeInfo // List files (blob) in a tree diff --git a/stash/client_repository_tree.go b/stash/client_repository_tree.go index 86edeecc..1fec807a 100644 --- a/stash/client_repository_tree.go +++ b/stash/client_repository_tree.go @@ -32,12 +32,6 @@ type TreeClient struct { ref gitprovider.RepositoryRef } -// Create creates,updates,deletes a tree -func (c *TreeClient) Create(ctx context.Context, tree *gitprovider.TreeInfo) (*gitprovider.TreeInfo, error) { - return nil, fmt.Errorf("error creaing tree %s. not implemented in stash yet", tree.SHA) - -} - // Get returns a tree func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) { return nil, fmt.Errorf("error getting tree %s. not implemented in stash yet", sha)