moved filters into each provider.

Readme.md

@@ -1,22 +1,22 @@
 # GoGitBackup
 
 This is a utility to download and backup all your GitHub/GitLab accounts to disk.
-It uses the APIs of each provider to find all the reposeories you have access to and clones or pulls them into the same root directory. 
+It uses the APIs of each provider to find all the repositories you have access to and clones or pulls them into the same root directory. 
 
 This tool is intended for local backups of all __your__ work. The tool uses the git-provider API to list all repositories you have access to and clones them. It is intended to backup work from internal Github/GitLab repositories in case you lose access to them, e.g., in case you leave university.
 
 ## Usage
-The tool is based of a `*.yml` config file where you can define one or more accounts to back up.
+The tool is based on a `*.yml` config file where you can define one or more accounts to back up.
 
 ```
 NAME:
-   gitback - Utility to backup your git(Hub|lab) accounts.
+   GoGitBackup - Utility to backup your git(Hub|lab) accounts.
 
 USAGE:
-   backup-git [global options] command [command options] [arguments...]
+   GoGitBackup [global options] command [command options] [arguments...]
 
 COMMANDS:
-   backup, b  performs backup of all git(hub/lab) accounts that can be accessed.
+   backup, b  performs a backup of all git(hub/lab) accounts that can be accessed.
    check, c   check what we can backup using this utility and also validates your config ;)
    help, h    Shows a list of commands or help for one command
 
@@ -28,9 +28,9 @@ GLOBAL OPTIONS:
 ```
 
 ### Config
-In order to run the utily, you need to specify at least one accont and a local reposetory. 
+To run the utility, you need to specify at least one account and a local repository. 
 An exemplary config file can look like this:
-```
+```yml
 repository: /tmp/test
 accounts:
   - name: Personal GitHub
@@ -43,57 +43,43 @@ The following accounts are supported:
 
 #### GitHub
 For GitHub, you need to specify the following fields in the config file:
-```
+```yml
   - name: <A Name of this account for logging>
     token: <a github access token>
     provider: 0
     args:
       - <the github usename of the token>
 ```
 
-Inorder to obtain a github token follow the this guid ()[]. 
-#### Filters
-Sometimes you want or need to avoid some reposetories, for that GoGitBackup has the ability to add filters.
-Each filter is added to the `filterList` propertie to the config. For the filter implementation we use (Tengo)[github.com/d5/tengo/v2] Script. Each filter is executed in sequence, each filter can use the following variabels.
-
-| name | description |
-| ---- | ----------- |
-| owner | bool - true if the reposetory is owned by the git user. | 
-| member| bool - true if the git user is a member of reposetory. |
-| visibility | int - Public = 0, Private = 1, Internal = 2 | 
-| size | int - size of the reposetory | 
-| name | string - name of the reposetory |
-
-Each script needs to set a variable `r`, for example, `r := owner` checks if the resposetory is owned by the token owner. 
-##### Planed Features
-
- - [ ] Store all Issues
- - [ ] Store the Wiki
- - [ ] Store all releases
+In order to obtain a GitHub token, follow this guide  [guid](https://docs.github.com/en/github/authenticating-to-github/keeping-your-account-and-data-secure/creating-a-personal-access-token).
  
  #### GitLab
  For GitLab, you need to specify the following fields in the config file:
- ```
+```yml
      - name: <A Name of this account for logging>
        token: <Gitlab API Token>
        provider: 1
        args:
-         - <base-url of your gitlab installation, optional will use gitlab.com by default>
+         - <base-url of your GitLab installation, optional will use gitlab.com by default>
  ```
  
- Inorder to obtain a gitlab token follow the this guid ()[]. 
-
-##### Planed Features
+ In order to obtain a GitLab token, follow this [guid](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html). 
 
- - [ ] Store all Issues
- - [ ] Store the Wiki
+### Filters
+Sometimes you want or need to avoid some repositories. For that GoGitBackup can add filters.
+Each filter is added to the `filterList` property of each provider config. 
+For the filter implementation we use [Tengo](github.com/d5/tengo/v2) Script. Each filter is executed in sequence. Each filter can use the following variables.
 
-## Development
-The tool is based on go-lang 1.13 and should be easily extendable to other git-as-a-service providers, PR's welcome.
-
-### Planed Features
- - [ ] Enable different git-authentication methods
- - [ ] Enable Error Resolution for failing pulls (clone/overwrite)
- - [x] Improve the output text to be more helpful
- - [x] Add a progress bar
+| name | description |
+| ---- | ----------- |
+| owner | bool - true if the reposetory is owned by the git user. | 
+| member| bool - true if the git user is a member of reposetory. |
+| visibility | int - Public = 0, Private = 1, Internal = 2 | 
+| size | int - size of the reposetory | 
+| name | string - name of the reposetory |
 
+Each script needs to set a variable `r`; for example, `r := owner` checks if the repository is owned by the token owner. 
+
+
+## Development
+The tool is based on go-lang 1.13 and should be easily extendable to other git-as-a-service providers, PR's welcome.

backup/backup.go

@@ -42,19 +42,18 @@ type Account struct {
 	Token     string   `yaml:"token"`
 	Args      []string `yaml:"args"`
 	BlackList []string `yaml:"blacklist"`
+	FilterList []string  `yaml:"filters"`
 }
 
 type Config struct {
 	Repository string    `yaml:"repository"`
 	Accounts   []Account `yaml:"accounts"`
-	FilterList []string  `yaml:"filters"`
 }
 
 type GoGitBackup struct {
 	clients []client
 	config  *Config
 	repos   []Repository
-	filters []*tengo.Script
 }
 
 type Visibility int
@@ -79,6 +78,7 @@ type client interface {
 	Init() error
 	List() ([]Repository, error)
 	Name() string
+	RegisterFilter(filters []*tengo.Script)
 }
 
 func _info(bar *pb.ProgressBar, msg string) {
@@ -104,41 +104,42 @@ func NewGoBackup(cnf *Config) (*GoGitBackup, error) {
 	clients := make([]client, 0)
 
 	for _, account := range cnf.Accounts {
+		filters := make([]*tengo.Script, 0)
+		for _, filterCode := range account.FilterList {
+			filters = append(filters, tengo.NewScript([]byte(filterCode)))
+		}
+
 		switch account.Provider {
 
-		case GitHub:
-			client := &_githubClient{
-				ctx:   context.Background(),
-				Token: account.Token,
-				name:  account.Name,
-			}
-			if account.Args != nil && len(account.Args) > 0 {
-				client.User = account.Args[0]
-			}
-			clients = append(clients, client)
-
-		case GitLab:
-			client := &_gitlabClient{
-				Token: account.Token,
-				name:  account.Name,
-			}
-			if account.Args != nil && len(account.Args) > 0 {
-				client.BaseURL = account.Args[0]
-			}
-			clients = append(clients, client)
-			//TODO: extend here if you add a new provider
+			case GitHub:
+				client := &_githubClient{
+					ctx:   context.Background(),
+					Token: account.Token,
+					name:  account.Name,
+				}
+				if account.Args != nil && len(account.Args) > 0 {
+					client.User = account.Args[0]
+				}
+				client.RegisterFilter(filters)
+				clients = append(clients, client)
+
+			case GitLab:
+				client := &_gitlabClient{
+					Token: account.Token,
+					name:  account.Name,
+				}
+				if account.Args != nil && len(account.Args) > 0 {
+					client.BaseURL = account.Args[0]
+				}
+				client.RegisterFilter(filters)
+				clients = append(clients, client)
+				//TODO: extend here if you add a new provider
 		}
 	}
 
-	filters := make([]*tengo.Script, 0)
-	for _, filterCode := range cnf.FilterList {
-		filters = append(filters, tengo.NewScript([]byte(filterCode)))
-	}
-
 	return &GoGitBackup{
 		config:  cnf,
 		clients: clients,
-		filters: filters,
 	}, nil
 }
 
@@ -210,16 +211,6 @@ func (c *GoGitBackup) Check() error {
 		repos = append(repos, repo...)
 	}
 
-	filtered := make([]Repository, 0)
-	for _, repo := range repos {
-		//make this a function...
-		if c.filter(repo) {
-			filtered = append(filtered, repo)
-		}
-	}
-
-	c.repos = filtered
-
 	fmt.Printf("Found the following repositories:\n")
 	fmt.Printf("| %60.10s\t| %10.10s\t| %10.10s\t|\n", "Name", "CreatedAt", "Size")
 
@@ -230,8 +221,8 @@ func (c *GoGitBackup) Check() error {
 	return nil
 }
 
-func (c *GoGitBackup) filter(repo Repository) bool {
-	for _, filter := range c.filters {
+func filter(repo Repository,filters []*tengo.Script) bool {
+	for _, filter := range filters {
 		if !apply(filter, repo) {
 			return false
 		}

backup/github.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/google/go-github/v28/github"
 	"golang.org/x/oauth2"
-
+	"github.com/d5/tengo/v2"
 	"strings"
 )
 
@@ -16,6 +16,7 @@ type _githubClient struct {
 	Token  string
 	User   string
 	name   string
+	filters []*tengo.Script
 }
 
 func (c *_githubClient) Name() string {
@@ -38,6 +39,10 @@ func (c *_githubClient) Init() error {
 	return nil
 }
 
+func (c *_githubClient) RegisterFilter(filters []*tengo.Script)  {
+	c.filters = filters
+}
+
 func (c *_githubClient) List() ([]Repository, error) {
 
 	list, res, err := c.client.Repositories.List(c.ctx, "", &github.RepositoryListOptions{
@@ -70,18 +75,21 @@ func (c *_githubClient) List() ([]Repository, error) {
 
 		owner := repo != nil && repo.Owner != nil && repo.Owner.Name != nil && *repo.Owner.Name == c.User
 
-		repoList = append(repoList,
-			Repository{
-				CloneUrl:   url,
-				Name:       repo.GetFullName(),
-				Size:       int64(repo.GetSize()),
-				CreatedAt:  repo.GetCreatedAt().UTC(),
-				Owner:      owner,
-				Member:     true,
-				Visibility: visibility,
-			})
+		r := Repository{
+			CloneUrl:   url,
+			Name:       repo.GetFullName(),
+			Size:       int64(repo.GetSize()),
+			CreatedAt:  repo.GetCreatedAt().UTC(),
+			Owner:      owner,
+			Member:     true,
+			Visibility: visibility,
+		}
 
-	}
+		if filter(r, c.filters) {
+			repoList = append(repoList,r)
+		}
 
+	}
+
 	return repoList, nil
 }

backup/gitlab.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/xanzy/go-gitlab"
+	"github.com/d5/tengo/v2"
 )
 
 type _gitlabClient struct {
@@ -13,6 +14,7 @@ type _gitlabClient struct {
 	client  *gitlab.Client
 	name    string
 	user    *gitlab.User
+	filters []*tengo.Script
 }
 
 func (c *_gitlabClient) Init() error {
@@ -79,15 +81,19 @@ func (c *_gitlabClient) List() ([]Repository, error) {
 			break
 		}
 
-		repoList = append(repoList, Repository{
+		r := Repository{
 			CloneUrl:   strings.Replace(project.HTTPURLToRepo, "https://", fmt.Sprintf("https://oauth2:%s@", c.Token), -1),
 			Name:       strings.ReplaceAll(strings.ReplaceAll(project.NameWithNamespace, " / ", "/"), " ", "_"),
 			Size:       size,
 			CreatedAt:  *project.CreatedAt,
 			Owner:      project.Owner != nil && project.Owner.ID == c.user.ID,
 			Member:     isMember,
 			Visibility: visibility,
-		})
+		}
+
+		if filter(r,c.filters) {
+			repoList = append(repoList,r)
+		}
 	}
 
 	return repoList, nil
@@ -96,3 +102,6 @@ func (c *_gitlabClient) List() ([]Repository, error) {
 func (c *_gitlabClient) Name() string {
 	return c.name
 }
+func (c *_gitlabClient) RegisterFilter(filters []*tengo.Script)  {
+	c.filters = filters
+}