diff --git a/clone.go b/clone.go new file mode 100644 index 0000000..b49deb3 --- /dev/null +++ b/clone.go @@ -0,0 +1,401 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "sync" + "time" + + "github.com/bradleyfalzon/ghinstallation/v2" + "github.com/google/go-github/v57/github" + "github.com/qiniu/reviewbot/config" + "github.com/qiniu/reviewbot/internal/lintersutil" + "github.com/qiniu/x/log" + gitv2 "sigs.k8s.io/prow/pkg/git/v2" +) + +func (s *Server) prepareGitRepos(ctx context.Context, org, repo string, num int, platform config.Platform, installationID int64) (workspace string, workDir string, err error) { + log := lintersutil.FromContext(ctx) + workspace, err = prepareRepoDir(org, repo, num) + if err != nil { + log.Errorf("failed to prepare workspace: %v", err) + return "", "", err + } + + refs, workDir := s.fixRefs(workspace, org, repo) + log.Debugf("refs: %+v", refs) + for _, ref := range refs { + opt := gitv2.ClientFactoryOpts{ + CacheDirBase: github.String(s.repoCacheDir), + Persist: github.Bool(true), + } + + gitConfig := s.newGitConfigBuilder(ctx, ref.Org, ref.Repo, platform, installationID).Build() + log.Debugf("git config: %+v", gitConfig) + if err := s.configureGitAuth(&opt, gitConfig); err != nil { + return "", "", fmt.Errorf("failed to configure git auth: %w", err) + } + + log.Debugf("git options: %+v", opt) + gitClient, err := gitv2.NewClientFactory(opt.Apply) + if err != nil { + log.Errorf("failed to create git client factory: %v", err) + return "", "", err + } + + r, err := gitClient.ClientForWithRepoOpts(ref.Org, ref.Repo, gitv2.RepoOpts{ + CopyTo: ref.PathAlias, + }) + if err != nil { + log.Errorf("failed to clone for %s/%s: %v", ref.Org, ref.Repo, err) + return "", "", err + } + + // main repo, need to checkout PR and update submodules if any + if ref.Org == org && ref.Repo == repo { + if err := r.CheckoutPullRequest(num); err != nil { + log.Errorf("failed to checkout pull request %d: %v", num, err) + return "", "", err + } + + // update submodules if any + if err := updateSubmodules(ctx, r.Directory(), repo); err != nil { + log.Errorf("error updating submodules: %v", err) + // continue to run other linters + } + } + } + + return workspace, workDir, nil +} + +func updateSubmodules(ctx context.Context, repoDir, repo string) error { + log := lintersutil.FromContext(ctx) + gitModulesFile := path.Join(repoDir, ".gitmodules") + if _, err := os.Stat(gitModulesFile); err != nil { + if errors.Is(err, os.ErrNotExist) { + log.Infof("no .gitmodules file in repo %s", repo) + return nil + } + return err + } + + log.Info("git pull submodule in progress") + cmd := exec.Command("git", "submodule", "update", "--init", "--recursive") + cmd.Dir = repoDir + out, err := cmd.CombinedOutput() + if err != nil { + log.Errorf("error when git pull submodule: %v, output: %s", err, out) + return err + } + + log.Infof("git pull submodule output: %s", out) + return nil +} + +func (s *Server) fixRefs(workspace string, org, repo string) ([]config.Refs, string) { + var repoCfg config.RepoConfig + if v, ok := s.config.CustomRepos[org]; ok { + repoCfg = v + } + if v, ok := s.config.CustomRepos[org+"/"+repo]; ok { + repoCfg = v + } + + var mainRepoFound bool + var workDir string + refs := make([]config.Refs, 0, len(repoCfg.Refs)) + for _, ref := range repoCfg.Refs { + if ref.PathAlias != "" { + ref.PathAlias = filepath.Join(workspace, ref.PathAlias) + } else { + ref.PathAlias = filepath.Join(workspace, ref.Repo) + } + refs = append(refs, ref) + + if ref.Repo == repo && ref.Org == org { + mainRepoFound = true + workDir = ref.PathAlias + } + } + + if !mainRepoFound { + // always add the main repo to the list + workDir = filepath.Join(workspace, repo) + refs = append(refs, config.Refs{ + Org: org, + Repo: repo, + PathAlias: workDir, + }) + } + + return refs, workDir +} + +// GitHubAppAuth stores the authentication information for GitHub App +type GitHubAppAuth struct { + AppID int64 + InstallationID int64 + PrivateKeyPath string +} + +// GitAuth stores the authentication information for different platforms +type GitAuth struct { + // GitHub authentication + GitHubAccessToken string + GitHubAppAuth *GitHubAppAuth + GitHubOAuthToken string + + // GitLab authentication + GitLabAccessToken string + GitLabOAuthToken string +} + +// GitConfig stores the Git repository configuration +type GitConfig struct { + Platform config.Platform + Host string // gitlab/github host + Auth GitAuth +} + +// githubAppTokenCache implements the cache for GitHub App tokens +type githubAppTokenCache struct { + sync.RWMutex + tokens map[string]tokenWithExp // key: installationID, value: token + appAuth *GitHubAppAuth +} + +// GitConfigBuilder is used to build the Git configuration for a specific request +type GitConfigBuilder struct { + server *Server + org string + repo string + platform config.Platform + // installationID is the installation ID for the GitHub App + installationID int64 +} + +type tokenWithExp struct { + token string + exp time.Time +} + +// newGitHubAppTokenCache create a new token cache +func newGitHubAppTokenCache(appID int64, privateKeyPath string) *githubAppTokenCache { + return &githubAppTokenCache{ + tokens: make(map[string]tokenWithExp), + appAuth: &GitHubAppAuth{ + AppID: appID, + PrivateKeyPath: privateKeyPath, + }, + } +} + +func (c *githubAppTokenCache) getToken(org string, installationID int64) (string, error) { + c.RLock() + t, exists := c.tokens[org] + c.RUnlock() + + if exists && t.exp.After(time.Now()) { + return t.token, nil + } + + c.Lock() + defer c.Unlock() + + // double check + if t, exists := c.tokens[org]; exists { + if t.exp.After(time.Now()) { + return t.token, nil + } + } + + // get new token + token, err := c.refreshToken(installationID) + if err != nil { + return "", err + } + + log.Debugf("refreshed token for %s, installationID: %d", org, installationID) + // cache token, 1 hour expiry + c.tokens[org] = tokenWithExp{ + token: token, + // add a buffer to avoid token expired + exp: time.Now().Add(time.Hour - time.Minute), + } + + return token, nil +} + +// refreshToken refresh the GitHub App token +func (c *githubAppTokenCache) refreshToken(installationID int64) (string, error) { + tr, err := ghinstallation.NewKeyFromFile( + http.DefaultTransport, + c.appAuth.AppID, + installationID, + c.appAuth.PrivateKeyPath, + ) + if err != nil { + return "", fmt.Errorf("failed to create github app transport: %w", err) + } + + token, err := tr.Token(context.Background()) + if err != nil { + return "", fmt.Errorf("failed to get installation token: %w", err) + } + + return token, nil +} + +func (s *Server) newGitConfigBuilder(ctx context.Context, org, repo string, platform config.Platform, installationID int64) *GitConfigBuilder { + return &GitConfigBuilder{ + server: s, + org: org, + repo: repo, + platform: platform, + installationID: installationID, + } +} + +func (b *GitConfigBuilder) Build() GitConfig { + config := GitConfig{ + Platform: b.platform, + Host: b.getHostForPlatform(b.platform), + Auth: b.buildAuth(), + } + + return config +} + +func (b *GitConfigBuilder) getHostForPlatform(platform config.Platform) string { + switch platform { + case config.GitLab: + if b.server.gitLabHost != "" { + return b.server.gitLabHost + } + return "gitlab.com" + default: + return "github.com" + } +} + +func (b *GitConfigBuilder) buildAuth() GitAuth { + switch b.platform { + case config.GitHub: + return b.buildGitHubAuth() + case config.GitLab: + return b.buildGitLabAuth() + default: + return GitAuth{} + } +} + +func (b *GitConfigBuilder) buildGitHubAuth() GitAuth { + if b.server.gitHubAppAuth != nil { + appAuth := *b.server.gitHubAppAuth + appAuth.InstallationID = b.installationID + return GitAuth{ + GitHubAppAuth: &appAuth, + } + } + + if b.server.gitHubAccessToken != "" { + return GitAuth{ + GitHubAccessToken: b.server.gitHubAccessToken, + } + } + + if b.server.gitHubOAuthToken != "" { + return GitAuth{ + GitHubOAuthToken: b.server.gitHubOAuthToken, + } + } + + return GitAuth{} +} + +func (b *GitConfigBuilder) buildGitLabAuth() GitAuth { + if b.server.gitLabAccessToken != "" { + return GitAuth{ + GitLabAccessToken: b.server.gitLabAccessToken, + } + } + + if b.server.gitLabOAuthToken != "" { + return GitAuth{ + GitLabOAuthToken: b.server.gitLabOAuthToken, + } + } + + return GitAuth{} +} + +func (s *Server) configureGitAuth(opt *gitv2.ClientFactoryOpts, gConf GitConfig) error { + opt.Host = gConf.Host + log.Debugf("configure git auth for %s, platform: %s", gConf.Host, gConf.Platform) + switch gConf.Platform { + case config.GitHub: + return s.configureGitHubAuth(opt, gConf) + case config.GitLab: + return s.configureGitLabAuth(opt, gConf) + default: + return fmt.Errorf("unsupported platform: %s", gConf.Platform) + } +} + +func (s *Server) configureGitHubAuth(opt *gitv2.ClientFactoryOpts, config GitConfig) error { + auth := config.Auth + + switch { + case auth.GitHubAppAuth != nil: + opt.Token = func(org string) (string, error) { + log.Debugf("get token for %s, installationID: %d", org, auth.GitHubAppAuth.InstallationID) + return s.githubAppTokenCache.getToken(org, auth.GitHubAppAuth.InstallationID) + } + return nil + + case auth.GitHubOAuthToken != "": + opt.Token = func(org string) (string, error) { + return auth.GitHubOAuthToken, nil + } + return nil + + case auth.GitHubAccessToken != "": + opt.Token = func(org string) (string, error) { + return auth.GitHubAccessToken, nil + } + return nil + } + + // default use ssh key if no auth + opt.UseSSH = github.Bool(true) + return nil +} + +func (s *Server) configureGitLabAuth(opt *gitv2.ClientFactoryOpts, config GitConfig) error { + auth := config.Auth + + switch { + case auth.GitLabOAuthToken != "": + opt.Token = func(org string) (string, error) { + return auth.GitLabOAuthToken, nil + } + return nil + + case auth.GitLabAccessToken != "": + opt.Token = func(org string) (string, error) { + return auth.GitLabAccessToken, nil + } + return nil + } + + // default use ssh key if no auth + opt.UseSSH = github.Bool(true) + return nil +} diff --git a/config/config.go b/config/config.go index 1145127..9f4c80a 100644 --- a/config/config.go +++ b/config/config.go @@ -271,7 +271,7 @@ func NewConfig(conf string) (Config, error) { return c, nil } -func (c Config) GetLinterConfig(org, repo, ln string, repoType RepoType) Linter { +func (c Config) GetLinterConfig(org, repo, ln string, repoType Platform) Linter { linter := Linter{ Enable: boolPtr(true), Modifier: NewBaseModifier(), @@ -434,11 +434,11 @@ const ( Quiet ReportType = "quiet" ) -type RepoType string +type Platform string const ( - GitLab RepoType = "gitlab" - GitHub RepoType = "github" + GitLab Platform = "GitLab" + GitHub Platform = "GitHub" ) func boolPtr(b bool) *bool { diff --git a/main.go b/main.go index dfb2069..1f0f680 100644 --- a/main.go +++ b/main.go @@ -68,6 +68,7 @@ type options struct { gitHubAppID int64 gitHubAppInstallationID int64 gitHubAppPrivateKey string + gitHubOAuthToken string // log storage dir for local storage logDir string @@ -116,10 +117,10 @@ func gatherOptions() options { fs.Int64Var(&o.gitHubAppID, "github.app-id", 0, "github app id") fs.Int64Var(&o.gitHubAppInstallationID, "github.app-installation-id", 0, "github app installation id") fs.StringVar(&o.gitHubAppPrivateKey, "github.app-private-key", "", "github app private key") - + fs.StringVar(&o.gitHubOAuthToken, "github.oauth-token", "", "github oauth token") // gitlab related fs.StringVar(&o.gitLabAccessToken, "gitlab.access-token", "", "personal gitlab access token") - fs.StringVar(&o.gitLabHost, "gitlab.host", "gitlab.com", "gitlab server") + fs.StringVar(&o.gitLabHost, "gitlab.host", "https://gitlab.com", "gitlab server") err := fs.Parse(os.Args[1:]) if err != nil { @@ -237,15 +238,24 @@ func main() { webhookSecret: []byte(o.webhookSecret), gitClientFactory: v2, config: cfg, - gitHubAccessToken: o.gitHubAccessToken, - gitLabAccessToken: o.gitLabAccessToken, - appID: o.gitHubAppID, - appPrivateKey: o.gitHubAppPrivateKey, debug: o.debug, serverAddr: o.serverAddr, repoCacheDir: o.codeCacheDir, kubeConfig: o.kubeConfig, gitLabHost: o.gitLabHost, + gitLabAccessToken: o.gitLabAccessToken, + gitHubAccessToken: o.gitHubAccessToken, + gitHubOAuthToken: o.gitHubOAuthToken, + } + + s.gitHubAppAuth = &GitHubAppAuth{ + AppID: o.gitHubAppID, + InstallationID: o.gitHubAppInstallationID, + PrivateKeyPath: o.gitHubAppPrivateKey, + } + + if s.gitHubAppAuth != nil { + s.githubAppTokenCache = newGitHubAppTokenCache(s.gitHubAppAuth.AppID, s.gitHubAppAuth.PrivateKeyPath) } go s.initDockerRunner() diff --git a/server.go b/server.go index aead60e..c52ff33 100644 --- a/server.go +++ b/server.go @@ -26,7 +26,6 @@ import ( "net/http" "os" "os/exec" - "path" "path/filepath" "runtime" "strconv" @@ -73,11 +72,16 @@ type Server struct { // support gitlab gitLabHost string gitLabAccessToken string + gitLabOAuthToken string // support github app model - appID int64 + // gitHubAppID int64 + // gitHubAppPrivateKey string + gitHubAppAuth *GitHubAppAuth gitHubAccessToken string - appPrivateKey string + gitHubOAuthToken string + + githubAppTokenCache *githubAppTokenCache } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -178,7 +182,7 @@ func (s *Server) serveGitLab(w http.ResponseWriter, r *http.Request) { func (s *Server) handleGitHubEvent(ctx context.Context, event *github.PullRequestEvent) error { info := &codeRequestInfo{ - platform: "github", + platform: config.GitHub, num: event.GetPullRequest().GetNumber(), org: event.GetRepo().GetOwner().GetLogin(), repo: event.GetRepo().GetName(), @@ -194,8 +198,7 @@ func (s *Server) handleGitHubEvent(ctx context.Context, event *github.PullReques log.Errorf("failed to list pull request files, status code: %d", resp.StatusCode) return ErrListFile } - - workspace, workDir, err := s.prepareGitRepos(ctx, info.org, info.repo, info.num) + workspace, workDir, err := s.prepareGitRepos(ctx, info.org, info.repo, info.num, config.GitHub, installationID) if err != nil { return err } @@ -213,7 +216,7 @@ func (s *Server) handleGitHubEvent(ctx context.Context, event *github.PullReques func (s *Server) handleGitLabEvent(ctx context.Context, event *gitlab.MergeEvent) error { info := &codeRequestInfo{ - platform: "gitlab", + platform: config.GitLab, num: event.ObjectAttributes.IID, org: event.Project.Namespace, repo: event.Project.Name, @@ -231,7 +234,7 @@ func (s *Server) handleGitLabEvent(ctx context.Context, event *gitlab.MergeEvent return ErrListFile } - workspace, workDir, err := s.prepareGitRepos(ctx, info.org, info.repo, info.num) + workspace, workDir, err := s.prepareGitRepos(ctx, info.org, info.repo, info.num, config.GitLab, 0) if err != nil { log.Errorf("prepare repo dir failed: %v", err) return ErrPrepareDir @@ -257,7 +260,7 @@ func (s *Server) handleGitLabEvent(ctx context.Context, event *gitlab.MergeEvent } type codeRequestInfo struct { - platform string + platform config.Platform num int org string repo string @@ -595,7 +598,7 @@ func (s *Server) processMergeRequestEvent(ctx context.Context, event *gitlab.Mer } func (s *Server) GitLabClient() *gitlab.Client { - git, err := gitlab.NewClient(s.gitLabAccessToken, gitlab.WithBaseURL("https://"+s.gitLabHost+"/")) + git, err := gitlab.NewClient(s.gitLabAccessToken, gitlab.WithBaseURL(s.gitLabHost)) if err != nil { log.Fatalf("Failed to create client: %v", err) } @@ -603,7 +606,7 @@ func (s *Server) GitLabClient() *gitlab.Client { } func (s *Server) githubAppClient(installationID int64) *github.Client { - tr, err := ghinstallation.NewKeyFromFile(httpcache.NewMemoryCacheTransport(), s.appID, installationID, s.appPrivateKey) + tr, err := ghinstallation.NewKeyFromFile(httpcache.NewMemoryCacheTransport(), s.gitHubAppAuth.AppID, installationID, s.gitHubAppAuth.PrivateKeyPath) if err != nil { log.Fatalf("failed to create github app transport: %v", err) } @@ -648,118 +651,6 @@ func prepareRepoDir(org, repo string, num int) (string, error) { return dir, nil } -func (s *Server) prepareGitRepos(ctx context.Context, org, repo string, num int) (workspace string, workDir string, err error) { - log := lintersutil.FromContext(ctx) - workspace, err = prepareRepoDir(org, repo, num) - if err != nil { - log.Errorf("failed to prepare workspace: %v", err) - return "", "", err - } - - refs, workDir := s.fixRefs(workspace, org, repo) - log.Debugf("refs: %+v", refs) - for _, ref := range refs { - opt := gitv2.ClientFactoryOpts{ - CacheDirBase: github.String(s.repoCacheDir), - Persist: github.Bool(true), - UseSSH: github.Bool(true), - Host: ref.Host, - } - gitClient, err := gitv2.NewClientFactory(opt.Apply) - if err != nil { - log.Errorf("failed to create git client factory: %v", err) - return "", "", err - } - - r, err := gitClient.ClientForWithRepoOpts(ref.Org, ref.Repo, gitv2.RepoOpts{ - CopyTo: ref.PathAlias, - }) - if err != nil { - log.Errorf("failed to clone for %s/%s: %v", ref.Org, ref.Repo, err) - return "", "", err - } - - // main repo, need to checkout PR and update submodules if any - if ref.Org == org && ref.Repo == repo { - if err := r.CheckoutPullRequest(num); err != nil { - log.Errorf("failed to checkout pull request %d: %v", num, err) - return "", "", err - } - - // update submodules if any - if err := updateSubmodules(ctx, r.Directory(), repo); err != nil { - log.Errorf("error updating submodules: %v", err) - // continue to run other linters - } - } - } - - return workspace, workDir, nil -} - -func updateSubmodules(ctx context.Context, repoDir, repo string) error { - log := lintersutil.FromContext(ctx) - gitModulesFile := path.Join(repoDir, ".gitmodules") - if _, err := os.Stat(gitModulesFile); err != nil { - if errors.Is(err, os.ErrNotExist) { - log.Infof("no .gitmodules file in repo %s", repo) - return nil - } - return err - } - - log.Info("git pull submodule in progress") - cmd := exec.Command("git", "submodule", "update", "--init", "--recursive") - cmd.Dir = repoDir - out, err := cmd.CombinedOutput() - if err != nil { - log.Errorf("error when git pull submodule: %v, output: %s", err, out) - return err - } - - log.Infof("git pull submodule output: %s", out) - return nil -} - -func (s *Server) fixRefs(workspace string, org, repo string) ([]config.Refs, string) { - var repoCfg config.RepoConfig - if v, ok := s.config.CustomRepos[org]; ok { - repoCfg = v - } - if v, ok := s.config.CustomRepos[org+"/"+repo]; ok { - repoCfg = v - } - - var mainRepoFound bool - var workDir string - refs := make([]config.Refs, 0, len(repoCfg.Refs)) - for _, ref := range repoCfg.Refs { - if ref.PathAlias != "" { - ref.PathAlias = filepath.Join(workspace, ref.PathAlias) - } else { - ref.PathAlias = filepath.Join(workspace, ref.Repo) - } - refs = append(refs, ref) - - if ref.Repo == repo && ref.Org == org { - mainRepoFound = true - workDir = ref.PathAlias - } - } - - if !mainRepoFound { - // always add the main repo to the list - workDir = filepath.Join(workspace, repo) - refs = append(refs, config.Refs{ - Org: org, - Repo: repo, - PathAlias: workDir, - }) - } - - return refs, workDir -} - // check kubectl installed. func checkKubectlInstalled() error { cmd := exec.Command("kubectl", "version", "--client")