If applied, this will add options including recursive functionality to File Client Get(Gitlab), and add Tree Client with create,get,and list functionalities

github/client_repository_file.go

@@ -34,13 +34,20 @@ type FileClient struct {
 	ref gitprovider.RepositoryRef
 }
 
-// Get fetches and returns the contents of a file from a given branch and path
-func (c *FileClient) Get(ctx context.Context, path, branch string) ([]*gitprovider.CommitFile, error) {
+// Get fetches and returns the contents of a file or multiple files in a directory from a given branch and path with possible options of FilesGetOption
+// If a file path is given, the contents of the file are returned
+// If a directory path is given, the contents of the files in the path's root are returned
+func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) {
 
 	opts := &github.RepositoryContentGetOptions{
 		Ref: branch,
 	}
 
+	fileOpts := gitprovider.FilesGetOptions{}
+	for _, opt := range optFns {
+		opt.ApplyFilesGetOptions(&fileOpts)
+	}
+
 	_, directoryContent, _, err := c.c.Client().Repositories.GetContents(ctx, c.ref.GetIdentity(), c.ref.GetRepository(), path, opts)
 	if err != nil {
 		return nil, err
@@ -62,15 +69,14 @@ func (c *FileClient) Get(ctx context.Context, path, branch string) ([]*gitprovid
 		if err != nil {
 			return nil, err
 		}
-		err = output.Close()
-		if err != nil {
-			return nil, err
-		}
+		defer output.Close()
+
 		contentStr := string(content)
 		files = append(files, &gitprovider.CommitFile{
 			Path:    filePath,
 			Content: &contentStr,
 		})
+
 	}
 
 	return files, nil

github/client_repository_tree.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package github
+
+import (
+	"context"
+	"strings"
+
+	"github.com/fluxcd/go-git-providers/gitprovider"
+)
+
+// TreeClient implements the gitprovider.TreeClient interface.
+var _ gitprovider.TreeClient = &TreeClient{}
+
+// TreeClient operates on the trees in a specific repository.
+type TreeClient struct {
+	*clientContext
+	ref gitprovider.RepositoryRef
+}
+
+// Get returns a single tree using the SHA1 value for that tree.
+// uses https://docs.github.com/en/rest/git/trees#get-a-tree
+func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) {
+	// GET /repos/{owner}/{repo}/git/trees
+	repoName := c.ref.GetRepository()
+	repoOwner := c.ref.GetIdentity()
+	githubTree, _, err := c.c.Client().Git.GetTree(ctx, repoOwner, repoName, sha, recursive)
+	if err != nil {
+		return nil, err
+	}
+
+	treeEntries := make([]*gitprovider.TreeEntry, len(githubTree.Entries))
+	for ind, treeEntry := range githubTree.Entries {
+		size := 0
+		if *treeEntry.Type != "tree" {
+			size = *treeEntry.Size
+		}
+		treeEntries[ind] = &gitprovider.TreeEntry{
+			Path: *treeEntry.Path,
+			Mode: *treeEntry.Mode,
+			Type: *treeEntry.Type,
+			Size: size,
+			SHA:  *treeEntry.SHA,
+			URL:  *treeEntry.URL,
+		}
+	}
+
+	treeInfo := gitprovider.TreeInfo{
+		SHA:       *githubTree.SHA,
+		Tree:      treeEntries,
+		Truncated: *githubTree.Truncated,
+	}
+
+	return &treeInfo, nil
+
+}
+
+// List files (blob) in a tree givent the tree sha (path is not used with Github Tree client)
+func (c *TreeClient) List(ctx context.Context, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) {
+	treeInfo, err := c.Get(ctx, sha, recursive)
+	if err != nil {
+		return nil, err
+	}
+	treeEntries := make([]*gitprovider.TreeEntry, 0)
+	for _, treeEntry := range treeInfo.Tree {
+		if treeEntry.Type == "blob" {
+			if path == "" || (path != "" && strings.HasPrefix(treeEntry.Path, path)) {
+				treeEntries = append(treeEntries, &gitprovider.TreeEntry{
+					Path: treeEntry.Path,
+					Mode: treeEntry.Mode,
+					Type: treeEntry.Type,
+					Size: treeEntry.Size,
+					SHA:  treeEntry.SHA,
+					URL:  treeEntry.URL,
+				})
+			}
+		}
+	}
+
+	return treeEntries, nil
+}

github/integration_test.go

@@ -562,6 +562,97 @@ var _ = Describe("GitHub Provider", func() {
 
 	})
 
+	It("should be possible to get and list repo tree", func() {
+
+		userRepoRef := newUserRepoRef(testUser, testUserRepoName)
+
+		userRepo, err := c.UserRepositories().Get(ctx, userRepoRef)
+		Expect(err).ToNot(HaveOccurred())
+
+		defaultBranch := userRepo.Get().DefaultBranch
+
+		path0 := "clustersDir/cluster/machine.yaml"
+		content0 := "machine yaml content"
+		path1 := "clustersDir/cluster/machine1.yaml"
+		content1 := "machine1 yaml content"
+		path2 := "clustersDir/cluster2/clusterSubDir/machine2.yaml"
+		content2 := "machine2 yaml content"
+
+		files := []gitprovider.CommitFile{
+			{
+				Path:    &path0,
+				Content: &content0,
+			},
+			{
+				Path:    &path1,
+				Content: &content1,
+			},
+			{
+				Path:    &path2,
+				Content: &content2,
+			},
+		}
+
+		commitFiles := make([]gitprovider.CommitFile, 0)
+		for _, file := range files {
+			path := file.Path
+			content := file.Content
+			commitFiles = append(commitFiles, gitprovider.CommitFile{
+				Path:    path,
+				Content: content,
+			})
+		}
+
+		commit, err := userRepo.Commits().Create(ctx, *defaultBranch, "added files", commitFiles)
+		Expect(err).ToNot(HaveOccurred())
+		commitSha := commit.Get().Sha
+
+		// get tree
+		tree, err := userRepo.Trees().Get(ctx, commitSha, true)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Tree should have length 9 for : LICENSE, README.md, 3 blob (files), 4 tree (directories)
+		Expect(tree.Tree).To(HaveLen(9))
+
+		// itemsToBeIgnored initially with 2 for LICENSE and README.md, and will also include tree types
+		itemsToBeIgnored := 2
+		for ind, treeEntry := range tree.Tree {
+			if treeEntry.Type == "blob" {
+				if treeEntry.Path == "LICENSE" || treeEntry.Path == "README.md" {
+					continue
+				}
+				Expect(treeEntry.Path).To(Equal(*files[ind-itemsToBeIgnored].Path))
+				continue
+
+			}
+			itemsToBeIgnored += 1
+		}
+
+		// List tree items with no path provided
+		treeEntries, err := userRepo.Trees().List(ctx, commitSha, "", true)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Tree Entries should have length 5 for : LICENSE, README.md, 3 blob (files)
+		Expect(treeEntries).To(HaveLen(5))
+		for ind, treeEntry := range treeEntries {
+			if treeEntry.Path == "LICENSE" || treeEntry.Path == "README.md" {
+				continue
+			}
+			Expect(treeEntry.Path).To(Equal(*files[ind-2].Path))
+		}
+
+		//List tree items with path provided to filter on
+		treeEntries, err = userRepo.Trees().List(ctx, commitSha, "clustersDir/", true)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Tree Entries should have length 3 for :3 blob (files)
+		Expect(treeEntries).To(HaveLen(3))
+		for ind, treeEntry := range treeEntries {
+			Expect(treeEntry.Path).To(Equal(*files[ind].Path))
+		}
+
+	})
+
 	AfterSuite(func() {
 		if os.Getenv("SKIP_CLEANUP") == "1" {
 			return

github/resource_repository.go

@@ -78,6 +78,10 @@ func newUserRepository(ctx *clientContext, apiObj *github.Repository, ref gitpro
 			clientContext: ctx,
 			ref:           ref,
 		},
+		trees: &TreeClient{
+			clientContext: ctx,
+			ref:           ref,
+		},
 	}
 }
 
@@ -95,6 +99,7 @@ type userRepository struct {
 	branches     *BranchClient
 	pullRequests *PullRequestClient
 	files        *FileClient
+	trees        *TreeClient
 }
 
 func (r *userRepository) Get() gitprovider.RepositoryInfo {
@@ -137,6 +142,10 @@ func (r *userRepository) Files() gitprovider.FileClient {
 	return r.files
 }
 
+func (r *userRepository) Trees() gitprovider.TreeClient {
+	return r.trees
+}
+
 // Update will apply the desired state in this object to the server.
 // Only set fields will be respected (i.e. PATCH behaviour).
 // In order to apply changes to this object, use the .Set({Resource}Info) error

gitlab/client_repository_file.go

@@ -35,12 +35,21 @@ type FileClient struct {
 	ref gitprovider.RepositoryRef
 }
 
-// Get fetches and returns the contents of a file from a given branch and path
-func (c *FileClient) Get(_ context.Context, path, branch string) ([]*gitprovider.CommitFile, error) {
+// Get fetches and returns the contents of a file or multiple files in a directory from a given branch and path with possible options of FilesGetOption
+// If a file path is given, the contents of the file are returned
+// If a directory path is given, the contents of the files in the path's root are returned
+func (c *FileClient) Get(ctx context.Context, path, branch string, optFns ...gitprovider.FilesGetOption) ([]*gitprovider.CommitFile, error) {
+
+	filesGetOpts := gitprovider.FilesGetOptions{}
+
+	for _, opt := range optFns {
+		opt.ApplyFilesGetOptions(&filesGetOpts)
+	}
 
 	opts := &gitlab.ListTreeOptions{
-		Path: &path,
-		Ref:  &branch,
+		Path:      &path,
+		Ref:       &branch,
+		Recursive: &filesGetOpts.Recursive,
 	}
 
 	listFiles, _, err := c.c.Client().Repositories.ListTree(getRepoPath(c.ref), opts)
@@ -54,6 +63,9 @@ func (c *FileClient) Get(_ context.Context, path, branch string) ([]*gitprovider
 
 	files := make([]*gitprovider.CommitFile, 0)
 	for _, file := range listFiles {
+		if file.Type == "tree" {
+			continue
+		}
 		fileDownloaded, _, err := c.c.Client().RepositoryFiles.GetFile(getRepoPath(c.ref), file.Path, fileOpts)
 		if err != nil {
 			return nil, err

gitlab/client_repository_tree.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2020 The Flux CD contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gitlab
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/fluxcd/go-git-providers/gitprovider"
+	"github.com/xanzy/go-gitlab"
+)
+
+// TreeClient implements the gitprovider.TreeClient interface.
+var _ gitprovider.TreeClient = &TreeClient{}
+
+// TreeClient operates on the trees in a specific repository.
+type TreeClient struct {
+	*clientContext
+	ref gitprovider.RepositoryRef
+}
+
+// Get returns a tree
+func (c *TreeClient) Get(ctx context.Context, sha string, recursive bool) (*gitprovider.TreeInfo, error) {
+	return nil, fmt.Errorf("error getting tree %s. not implemented in gitlab yet", sha)
+
+}
+
+// List files (blob) in a tree, sha is represented by the branch name
+func (c *TreeClient) List(ctx context.Context, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) {
+	opts := &gitlab.ListTreeOptions{
+		Path:      &path,
+		Ref:       &sha,
+		Recursive: &recursive,
+	}
+
+	treeFiles, _, err := c.c.Client().Repositories.ListTree(getRepoPath(c.ref), opts)
+	if err != nil {
+		return nil, err
+	}
+
+	treeEntries := make([]*gitprovider.TreeEntry, 0)
+	for _, treeEntry := range treeFiles {
+		if treeEntry.Type == "blob" {
+			size := 0
+			treeEntries = append(treeEntries, &gitprovider.TreeEntry{
+				Path: treeEntry.Path,
+				Mode: treeEntry.Mode,
+				Type: treeEntry.Type,
+				Size: size,
+				ID:   treeEntry.ID,
+			})
+		}
+	}
+
+	return treeEntries, nil
+}