From b3adf8d6d228218323091936297051825d572ed2 Mon Sep 17 00:00:00 2001 From: David Roe Date: Mon, 20 Nov 2023 16:35:33 +0000 Subject: [PATCH] fix: fixes from testing --- internal/commands/artifact/run.go | 4 + .../commands/process/gitrepository/context.go | 27 +- .../process/gitrepository/gitrepository.go | 28 +- internal/git/branch.go | 14 +- internal/git/checkout.go | 102 ------- internal/git/clone.go | 162 ----------- internal/git/commit_list.go | 196 ------------- internal/git/commits.go | 33 ++- internal/git/defunct_cleanup.go | 59 ---- internal/git/diff_test.go | 6 +- internal/git/git.go | 43 ++- internal/git/{tree_test.go => git_test.go} | 0 internal/git/remote.go | 4 + internal/git/renames.go | 89 ------ internal/git/tree.go | 270 +++--------------- internal/report/output/output.go | 2 +- 16 files changed, 145 insertions(+), 894 deletions(-) delete mode 100644 internal/git/clone.go delete mode 100644 internal/git/commit_list.go delete mode 100644 internal/git/defunct_cleanup.go rename internal/git/{tree_test.go => git_test.go} (100%) delete mode 100644 internal/git/renames.go diff --git a/internal/commands/artifact/run.go b/internal/commands/artifact/run.go index c65f4a075..357243b59 100644 --- a/internal/commands/artifact/run.go +++ b/internal/commands/artifact/run.go @@ -290,6 +290,10 @@ func Run(ctx context.Context, opts flag.Options) (err error) { return fmt.Errorf("failed to get git context: %w", err) } + if opts.Diff && gitContext == nil { + return errors.New("--diff option requires a git repository") + } + if !opts.Quiet { outputhandler.StdErrLog("Loading rules") } diff --git a/internal/commands/process/gitrepository/context.go b/internal/commands/process/gitrepository/context.go index 2889afe87..8e3ad309d 100644 --- a/internal/commands/process/gitrepository/context.go +++ b/internal/commands/process/gitrepository/context.go @@ -11,6 +11,7 @@ import ( "github.com/google/go-github/github" "github.com/rs/zerolog/log" "golang.org/x/oauth2" + "gopkg.in/yaml.v3" "github.com/bearer/bearer/internal/flag" "github.com/bearer/bearer/internal/git" @@ -33,11 +34,15 @@ type Context struct { } func NewContext(options *flag.Options) (*Context, error) { - rootDir := git.GetRoot(options.Target) - if rootDir == "" { + if options.IgnoreGit { return nil, nil } + rootDir, err := git.GetRoot(options.Target) + if rootDir == "" || err != nil { + return nil, err + } + currentBranch, err := getCurrentBranch(options, rootDir) if err != nil { return nil, fmt.Errorf("error getting current branch name: %w", err) @@ -48,7 +53,7 @@ func NewContext(options *flag.Options) (*Context, error) { return nil, fmt.Errorf("error getting default branch name: %w", err) } - baseBranch, err := getBaseBranch(options) + baseBranch, err := getBaseBranch(options, defaultBranch) if err != nil { return nil, fmt.Errorf("error getting base branch name: %w", err) } @@ -87,7 +92,7 @@ func NewContext(options *flag.Options) (*Context, error) { fullName = urlInfo.FullName } - return &Context{ + context := &Context{ RootDir: rootDir, CurrentBranch: currentBranch, DefaultBranch: defaultBranch, @@ -101,7 +106,12 @@ func NewContext(options *flag.Options) (*Context, error) { Name: name, FullName: fullName, HasUncommittedChanges: hasUncommittedChanges, - }, nil + } + + contextYAML, _ := yaml.Marshal(context) + log.Debug().Msgf("git context:\n%s", contextYAML) + + return context, nil } func getCurrentBranch(options *flag.Options, rootDir string) (string, error) { @@ -125,7 +135,7 @@ func getDefaultBranch(options *flag.Options, rootDir string) (string, error) { return git.GetDefaultBranch(rootDir) } -func getBaseBranch(options *flag.Options) (string, error) { +func getBaseBranch(options *flag.Options, defaultBranch string) (string, error) { if !options.Diff { return "", nil } @@ -134,6 +144,11 @@ func getBaseBranch(options *flag.Options) (string, error) { return options.DiffBaseBranch, nil } + if defaultBranch != "" { + log.Debug().Msgf("using default branch %s for diff base branch", defaultBranch) + return defaultBranch, nil + } + return "", errors.New( "couldn't determine base branch for diff scanning. " + "please set the 'BEARER_DIFF_BASE_BRANCH' environment variable", diff --git a/internal/commands/process/gitrepository/gitrepository.go b/internal/commands/process/gitrepository/gitrepository.go index 05bb36519..d62a8dc2a 100644 --- a/internal/commands/process/gitrepository/gitrepository.go +++ b/internal/commands/process/gitrepository/gitrepository.go @@ -2,6 +2,7 @@ package gitrepository import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -21,7 +22,6 @@ import ( type Repository struct { ctx context.Context config settings.Config - rootPath, targetPath, gitTargetPath string context *Context @@ -38,7 +38,7 @@ func New(ctx context.Context, config settings.Config, targetPath string, context return nil, fmt.Errorf("failed to get relative target: %w", err) } - log.Debug().Msgf("git target: [%s/]%s", context.RootDir, gitTargetPath) + log.Debug().Msgf("git target: [%s]/%s", context.RootDir, gitTargetPath) repository := &Repository{ ctx: ctx, @@ -78,13 +78,13 @@ func (repository *Repository) fetchMergeBaseCommit() error { log.Debug().Msgf("merge base commit: %s", hash) - if git.CommitPresent(repository.rootPath, hash) { - return nil + if isPresent, err := git.CommitPresent(repository.context.RootDir, hash); isPresent || err != nil { + return err } log.Debug().Msgf("merge base commit not present, fetching") - if err := git.FetchRef(repository.ctx, repository.rootPath, hash); err != nil { + if err := git.FetchRef(repository.ctx, repository.context.RootDir, hash); err != nil { return err } @@ -97,9 +97,13 @@ func (repository *Repository) getCurrentFiles( ignore *ignore.FileIgnore, goclocResult *gocloc.Result, ) (*files.List, error) { + if repository.context.CurrentCommitHash == "" { + return &files.List{}, nil + } + var headFiles []files.File - gitFiles, err := git.ListTree(repository.rootPath, repository.context.CurrentCommitHash) + gitFiles, err := git.ListTree(repository.context.RootDir, repository.context.CurrentCommitHash) if err != nil { return nil, err } @@ -125,7 +129,7 @@ func (repository *Repository) getDiffFiles( renames := make(map[string]string) chunks := make(map[string]git.Chunks) - filePatches, err := git.Diff(repository.rootPath, repository.context.BaseCommitHash) + filePatches, err := git.Diff(repository.context.RootDir, repository.context.BaseCommitHash) if err != nil { return nil, err } @@ -176,7 +180,11 @@ func (repository *Repository) WithBaseBranch(body func() error) error { return nil } - if err := git.Switch(repository.rootPath, repository.context.BaseCommitHash, true); err != nil { + if repository.context.HasUncommittedChanges { + return errors.New("uncommitted changes found in your repository. commit or stash changes your changes and retry") + } + + if err := git.Switch(repository.context.RootDir, repository.context.BaseCommitHash, true); err != nil { return fmt.Errorf("error switching to base branch: %w", err) } @@ -196,10 +204,10 @@ func (repository *Repository) WithBaseBranch(body func() error) error { func (repository *Repository) restoreCurrent() error { if repository.context.CurrentBranch == "" { - return git.Switch(repository.rootPath, repository.context.CurrentCommitHash, true) + return git.Switch(repository.context.RootDir, repository.context.CurrentCommitHash, true) } - return git.Switch(repository.rootPath, repository.context.CurrentBranch, false) + return git.Switch(repository.context.RootDir, repository.context.CurrentBranch, false) } func (repository *Repository) fileFor( diff --git a/internal/git/branch.go b/internal/git/branch.go index 7c4a73981..80949858a 100644 --- a/internal/git/branch.go +++ b/internal/git/branch.go @@ -8,6 +8,10 @@ import ( func GetDefaultBranch(dir string) (string, error) { name, err := getRevParseAbbrevRef(dir, "origin/HEAD") if err != nil { + if strings.Contains(err.Error(), "unknown revision") { + return "", nil + } + return "", err } @@ -17,10 +21,18 @@ func GetDefaultBranch(dir string) (string, error) { // GetCurrentBranch gets the branch name. It is blank when detached. func GetCurrentBranch(dir string) (string, error) { name, err := getRevParseAbbrevRef(dir, "HEAD") - if name == "HEAD" || err != nil { + if err != nil { + if strings.Contains(err.Error(), "unknown revision") { + return "", nil + } + return "", err } + if name == "HEAD" { + return "", nil + } + return name, nil } diff --git a/internal/git/checkout.go b/internal/git/checkout.go index 03d9d3de9..9bf6d4cfd 100644 --- a/internal/git/checkout.go +++ b/internal/git/checkout.go @@ -2,7 +2,6 @@ package git import ( "context" - "fmt" ) func Switch(rootDir, ref string, detach bool) error { @@ -18,104 +17,3 @@ func Switch(rootDir, ref string, detach bool) error { append(args, ref)..., ) } - -func checkoutFiles(rootDir, ref string, filenames []string) error { - cmd := logAndBuildCommand( - context.TODO(), - "-c", - "advice.detachedHead=false", - "checkout", - ref, - "--pathspec-from-file=-", - "--pathspec-file-nul", - ) - cmd.Dir = rootDir - - stdin, err := cmd.StdinPipe() - if err != nil { - return err - } - - logWriter := &debugLogWriter{} - cmd.Stdout = logWriter - cmd.Stderr = logWriter - - if err := cmd.Start(); err != nil { - cmd.Cancel() //nolint:errcheck - return err - } - - for _, filename := range filenames { - _, err := stdin.Write([]byte(filename)) - if err != nil { - cmd.Cancel() //nolint:errcheck - return err - } - - _, err = stdin.Write([]byte{0}) - if err != nil { - cmd.Cancel() //nolint:errcheck - return err - } - } - - if err := stdin.Close(); err != nil { - cmd.Cancel() //nolint:errcheck - return err - } - - if err := cmd.Wait(); err != nil { - cmd.Cancel() //nolint:errcheck - return newError(err, logWriter.AllOutput()) - } - - // Using pathspec with checkout doesn't update the HEAD ref so do it manually - return basicCommand(context.TODO(), rootDir, "update-ref", "HEAD", ref) -} - -func fetchBlobsForRange(rootDir, firstCommitSHA, lastCommitSHA string, filenames []string) error { - objectIDs, err := getObjectIDsForRangeFiles(rootDir, firstCommitSHA, lastCommitSHA, filenames) - if err != nil { - return err - } - - return fetchBlobs(rootDir, objectIDs) -} - -// Fetches the given list of objects/blobs. -// -// There's no command in git that does this directly but we can get the desired -// behaviour by creating a pack and throwing it away -func fetchBlobs(rootDir string, objectIDs []string) error { - cmd := logAndBuildCommand(context.TODO(), "pack-objects", "--progress", "--stdout") - cmd.Dir = rootDir - - stdin, err := cmd.StdinPipe() - if err != nil { - return err - } - - logWriter := &debugLogWriter{} - cmd.Stderr = logWriter - - if err := cmd.Start(); err != nil { - cmd.Cancel() //nolint:errcheck - return err - } - - for _, objectID := range objectIDs { - fmt.Fprintln(stdin, objectID) - } - - if err := stdin.Close(); err != nil { - cmd.Cancel() //nolint:errcheck - return err - } - - if err := cmd.Wait(); err != nil { - cmd.Cancel() //nolint:errcheck - return newError(err, logWriter.AllOutput()) - } - - return nil -} diff --git a/internal/git/clone.go b/internal/git/clone.go deleted file mode 100644 index 499a90ea1..000000000 --- a/internal/git/clone.go +++ /dev/null @@ -1,162 +0,0 @@ -package git - -import ( - "context" - "errors" - "fmt" - "net/url" - "os" - "slices" - "time" - - "github.com/rs/zerolog/log" -) - -func CloneAndGetTree(token string, url *url.URL, branchName string) (*Tree, error) { - tempDir, err := os.MkdirTemp("", "tree") - if err != nil { - return nil, fmt.Errorf("failed to create temp dir: %s", err) - } - defer os.RemoveAll(tempDir) - - if err := cloneTree(tempDir, urlWithCredentials(url, token), branchName); err != nil { - return nil, fmt.Errorf("failed to clone: %s", err) - } - - return GetTree(tempDir) -} - -func CloneRangeAndCheckoutFiles( - token string, - url *url.URL, - branchName string, - previousCommit *CommitIdentifier, - commit CommitIdentifier, - filenames []string, - targetDir string, -) (bool, error) { - isRange, err := cloneTreeRange(urlWithCredentials(url, token), branchName, previousCommit, commit, targetDir) - if err != nil { - return false, err - } - - firstCommitSHA := commit.SHA - if isRange { - firstCommitSHA = previousCommit.SHA - } - - treeFiles, err := ListTree(targetDir, commit.SHA) - if err != nil { - return false, err - } - - filenames = appendMailmap(filenames, treeFiles) - - if err := fetchBlobsForRange(targetDir, firstCommitSHA, commit.SHA, filenames); err != nil { - return false, err - } - - // Remove remote to avoid accidentally fetching more data - if err := removeRemote(targetDir); err != nil { - return false, err - } - - if err := checkoutFiles(targetDir, commit.SHA, filenames); err != nil { - return false, err - } - - return isRange, nil -} - -func appendMailmap(filenames []string, treeFiles []TreeFile) []string { - if slices.Contains(filenames, mailmapFilename) { - return filenames - } - - for _, treeFile := range treeFiles { - if treeFile.Filename == mailmapFilename { - return append(filenames, mailmapFilename) - } - } - - return filenames -} - -func cloneTree(targetDir string, url *url.URL, branchName string) error { - return basicCommand( - context.TODO(), - "", - "clone", - "--depth=1", - "--branch", - branchName, - "--no-checkout", - "--no-tags", - "--filter=blob:none", - "--progress", - url.String(), - targetDir, - ) -} - -func cloneTreeRange( - url *url.URL, - branchName string, - previousCommit *CommitIdentifier, - commit CommitIdentifier, - targetDir string, -) (bool, error) { - if previousCommit == nil { - if err := cloneTreeSince(url, branchName, commit, targetDir); err != nil { - return false, err - } - } else { - if err := cloneTreeSince(url, branchName, *previousCommit, targetDir); err != nil { - return false, err - } - } - - if !CommitPresent(targetDir, commit.SHA) { - return false, errors.New("target commit not found") - } - - if previousCommit != nil && !CommitPresent(targetDir, previousCommit.SHA) { - log.Debug().Msg("previous commit is missing, possible re-written history") - // Fallback to non range clone - return false, nil - } - - return previousCommit != nil, nil -} - -func cloneTreeSince(url *url.URL, branchName string, commit CommitIdentifier, targetDir string) error { - cmd := logAndBuildCommand( - context.TODO(), - "clone", - "--shallow-since="+commit.Timestamp.Format(time.RFC3339), - "--branch", - branchName, - "--single-branch", - "--no-checkout", - "--no-tags", - "--filter=blob:none", - "--progress", - url.String(), - targetDir, - ) - - logWriter := &debugLogWriter{} - cmd.Stdout = logWriter - cmd.Stderr = logWriter - - if err := cmd.Run(); err != nil { - cmd.Cancel() //nolint:errcheck - return newError(err, logWriter.AllOutput()) - } - - return nil -} - -func removeRemote(rootDir string) error { - return basicCommand(context.TODO(), rootDir, "remote", "remove", "origin") -} diff --git a/internal/git/commit_list.go b/internal/git/commit_list.go deleted file mode 100644 index de4368745..000000000 --- a/internal/git/commit_list.go +++ /dev/null @@ -1,196 +0,0 @@ -package git - -import ( - "bufio" - "context" - "fmt" - "regexp" - "strings" - "time" -) - -var coAuthorValidPattern = regexp.MustCompile(`<.*>`) - -type CommitInfo struct { - CommitIdentifier - Committer string `json:"committer" yaml:"committer"` - Author string `json:"author" yaml:"author"` - CoAuthors []string `json:"co_authors" yaml:"co_authors"` -} - -func GetCommitList(rootDir, firstCommitSHA, lastCommitSHA string) ([]CommitInfo, error) { - separator := "---" - result := []CommitInfo{} - - cmd := logAndBuildCommand( - context.TODO(), - "log", - "--first-parent", - "--format=%H %cI%n%cN <%cE>%n%aN <%aE>%n%(trailers:unfold,valueonly,key=Co-authored-by)"+separator, - lastCommitSHA, - "--", - ) - cmd.Dir = rootDir - - logWriter := &debugLogWriter{} - cmd.Stderr = logWriter - - stdout, err := cmd.StdoutPipe() - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - if err := cmd.Start(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - scanner := bufio.NewScanner(stdout) - info := CommitInfo{} - n := 0 - - for scanner.Scan() { - line := scanner.Text() - if line == separator { - result = append(result, info) - - if info.SHA == firstCommitSHA { - break - } - - info = CommitInfo{} - n = 0 - continue - } - - switch n { - case 0: - splitLine := strings.SplitN(line, " ", 2) - - parsedTimestamp, err := time.Parse(time.RFC3339, splitLine[1]) - if err != nil { - return nil, err - } - - info.SHA = splitLine[0] - info.Timestamp = parsedTimestamp.UTC() - case 1: - info.Committer = line - case 2: - info.Author = line - default: - info.CoAuthors = append(info.CoAuthors, line) - } - - n++ - } - - if err := scanner.Err(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - stdout.Close() - - if err := cmd.Wait(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, newError(err, logWriter.AllOutput()) - } - - if err := translateCoAuthors(rootDir, result); err != nil { - return nil, err - } - - return result, nil -} - -func translateCoAuthors(rootDir string, commitList []CommitInfo) error { - translatedCoAuthors := make(map[string]string) - toTranslate := []string{} - - for _, commitInfo := range commitList { - for _, author := range commitInfo.CoAuthors { - translated := author - if coAuthorValidPattern.MatchString(author) { - translated = "" - toTranslate = append(toTranslate, author) - } - - translatedCoAuthors[author] = translated - } - } - - mailmapAuthors, err := checkMailmap(rootDir, toTranslate) - if err != nil { - return err - } - - for i, author := range toTranslate { - translatedCoAuthors[author] = mailmapAuthors[i] - } - - for i := range commitList { - commitInfo := &commitList[i] - - for j := range commitInfo.CoAuthors { - commitInfo.CoAuthors[j] = translatedCoAuthors[commitInfo.CoAuthors[j]] - } - } - - return nil -} - -func checkMailmap(rootDir string, authors []string) ([]string, error) { - cmd := logAndBuildCommand(context.TODO(), "check-mailmap", "--stdin") - cmd.Dir = rootDir - - stdin, err := cmd.StdinPipe() - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - go func() { - for _, author := range authors { - fmt.Fprintln(stdin, author) - } - - stdin.Close() - }() - - logWriter := &debugLogWriter{} - cmd.Stderr = logWriter - - stdout, err := cmd.StdoutPipe() - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - if err := cmd.Start(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - result := []string{} - scanner := bufio.NewScanner(stdout) - - for scanner.Scan() { - result = append(result, scanner.Text()) - } - - if err := scanner.Err(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - stdout.Close() - - if err := cmd.Wait(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, newError(err, logWriter.AllOutput()) - } - - return result, nil -} diff --git a/internal/git/commits.go b/internal/git/commits.go index c1877daa4..ef72eac0e 100644 --- a/internal/git/commits.go +++ b/internal/git/commits.go @@ -6,21 +6,25 @@ import ( ) // GetCurrentCommit gets a current commit from a HEAD for a local directory -func GetCurrentCommit(dir string) (string, error) { +func GetCurrentCommit(rootDir string) (string, error) { output, err := captureCommandBasic( context.TODO(), - dir, + rootDir, "rev-parse", "HEAD", ) + if err != nil && strings.Contains(err.Error(), "unknown revision") { + return "", nil + } + return strings.TrimSpace(output), err } -func GetMergeBase(dir string, ref1, ref2 string) (sha string, err error) { +func GetMergeBase(rootDir string, ref1, ref2 string) (string, error) { output, err := captureCommandBasic( context.TODO(), - dir, + rootDir, "merge-base", ref1, ref2, @@ -29,10 +33,21 @@ func GetMergeBase(dir string, ref1, ref2 string) (sha string, err error) { return strings.TrimSpace(output), err } -func CommitPresent(rootDir, sha string) bool { - cmd := logAndBuildCommand(context.TODO(), "cat-file", "-t", sha) - cmd.Dir = rootDir +func CommitPresent(rootDir, hash string) (bool, error) { + output, err := captureCommandBasic( + context.TODO(), + rootDir, + "cat-file", + "-t", + hash, + ) + if err != nil { + if strings.Contains(err.Error(), "could not get object info") { + return false, nil + } + + return false, err + } - output, _ := cmd.Output() - return string(output) == "commit\n" + return output == "commit\n", nil } diff --git a/internal/git/defunct_cleanup.go b/internal/git/defunct_cleanup.go deleted file mode 100644 index 1d443a2cc..000000000 --- a/internal/git/defunct_cleanup.go +++ /dev/null @@ -1,59 +0,0 @@ -package git - -import ( - "context" - "os/exec" - "strings" - "time" - - "github.com/rs/zerolog/log" -) - -// MonitorDefunct periodically cleans up defunct zombie git proccess that happen on git error and pile up over time -func MonitorDefunct(ctx context.Context) { - timer := time.NewTicker(5 * time.Second) - for { - select { - case <-timer.C: - cleanupDefunct() - case <-ctx.Done(): - return - } - } -} - -func cleanupDefunct() { - log.Debug().Msgf("[git] defunct checking for abandoned process") - - cmdProcess := exec.Command("ps", "-A", "-H") - stdout, err := cmdProcess.CombinedOutput() - if err != nil { - log.Debug().Msgf("failed to get output of command %s", err) - } - - defunctPIDs := make([]string, 0) - - lines := strings.Split(string(stdout), "\n") - - for _, line := range lines { - if !regexpDefunctProcess.MatchString(line) { - continue - } - - pid := regexpPID.FindString(line) - if pid == "" { - continue - } - - defunctPIDs = append(defunctPIDs, strings.Trim(pid, " ")) - } - - for _, pid := range defunctPIDs { - cmdKill := exec.Command("kill", pid) - if err := cmdKill.Run(); err != nil { - log.Debug().Msgf("failed to kill git process %s", err) - } - } - - log.Debug().Msgf("[git] defunct process cleanup found %d processes", len(defunctPIDs)) -} diff --git a/internal/git/diff_test.go b/internal/git/diff_test.go index a01a6b621..be02b104c 100644 --- a/internal/git/diff_test.go +++ b/internal/git/diff_test.go @@ -159,8 +159,8 @@ var _ = Describe("ChunkRange", func() { }) When("there are no lines in the range", func() { - It("returns an invalid (< start line) value", func() { - Expect(git.ChunkRange{LineNumber: 2, LineCount: 0}.EndLineNumber()).To(Equal(1)) + It("returns the start line number", func() { + Expect(git.ChunkRange{LineNumber: 2, LineCount: 0}.EndLineNumber()).To(Equal(2)) }) }) }) @@ -214,7 +214,7 @@ var _ = Describe("ChunkRange", func() { }) var _ = Describe("Chunks", func() { - FDescribe("TranslateRange", func() { + Describe("TranslateRange", func() { When("the base range is preceded by an add chunk", func() { chunks := git.Chunks{{ From: git.ChunkRange{LineNumber: 0, LineCount: 0}, diff --git a/internal/git/git.go b/internal/git/git.go index a63c6a208..6cd424cb3 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -5,19 +5,16 @@ import ( "context" "fmt" "io" - "net/url" "os" "os/exec" + "path" "regexp" "strconv" "strings" - "time" "github.com/rs/zerolog/log" -) -const ( - mailmapFilename = ".mailmap" + "github.com/bearer/bearer/internal/util/file" ) var ( @@ -27,24 +24,29 @@ var ( "GCM_INTERACTIVE=never", "GIT_LFS_SKIP_SMUDGE=1", ) +) - specialFiles []string = []string{ - mailmapFilename, - ".gitignore", - ".gitattributes", - ".gitmodules", +func GetRoot(targetPath string) (string, error) { + dir := targetPath + if !file.IsDir(dir) { + dir = path.Dir(dir) } -) -type CommitIdentifier struct { - SHA string `json:"sha" yaml:"sha"` - Timestamp time.Time `json:"timestamp" yaml:"timestamp"` -} + output, err := captureCommandBasic(context.TODO(), dir, "rev-parse", "--show-toplevel") + if err != nil { + if strings.Contains(err.Error(), "not a git repository") { + return "", nil + } -func urlWithCredentials(originalURL *url.URL, token string) *url.URL { - result := *originalURL - result.User = url.UserPassword("x", token) - return &result + return "", err + } + + path := strings.TrimSpace(output) + if path == "" { + return "", nil + } + + return file.CanonicalPath(path) } func logAndBuildCommand(ctx context.Context, args ...string) *exec.Cmd { @@ -118,9 +120,6 @@ func captureCommandBasic(ctx context.Context, workingDir string, args ...string) return } -var regexpDefunctProcess = regexp.MustCompile(" git ") -var regexpPID = regexp.MustCompile("[0-9]+ ") - func unquoteFilename(quoted string) (string, error) { if len(quoted) == 0 || quoted[0] != '"' { return quoted, nil diff --git a/internal/git/tree_test.go b/internal/git/git_test.go similarity index 100% rename from internal/git/tree_test.go rename to internal/git/git_test.go diff --git a/internal/git/remote.go b/internal/git/remote.go index 6b4141d58..f88f5f3b7 100644 --- a/internal/git/remote.go +++ b/internal/git/remote.go @@ -14,5 +14,9 @@ func GetOriginURL(dir string) (string, error) { "origin", ) + if err != nil && strings.Contains(err.Error(), "No such remote 'origin'") { + return "", nil + } + return strings.TrimSpace(output), err } diff --git a/internal/git/renames.go b/internal/git/renames.go deleted file mode 100644 index 7bd09f0ce..000000000 --- a/internal/git/renames.go +++ /dev/null @@ -1,89 +0,0 @@ -package git - -import ( - "bufio" - "context" - "fmt" - "strings" -) - -type RenamedFile struct { - PreviousFilename string `json:"previous_filename" yaml:"previous_filename"` - NewFilename string `json:"new_filename" yaml:"new_filename"` -} - -func GetRenames(rootDir, firstCommitSHA, lastCommitSHA string) ([]RenamedFile, error) { - cmd := logAndBuildCommand( - context.TODO(), - "log", - "--first-parent", - "--find-renames", - "--break-rewrites", - "--name-status", - "--diff-filter=R", - "--pretty=tformat:", - firstCommitSHA+".."+lastCommitSHA, - "--", - ) - cmd.Dir = rootDir - - logWriter := &debugLogWriter{} - cmd.Stderr = logWriter - - stdout, err := cmd.StdoutPipe() - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - if err := cmd.Start(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - scanner := bufio.NewScanner(stdout) - - renameMap := make(map[string]string) - - for scanner.Scan() { - line := scanner.Text() - splitLine := strings.Split(line, "\t") - - prevFilename, err := unquoteFilename(splitLine[1]) - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, fmt.Errorf("failed to unquote previous filename: %s", err) - } - newFilename, err := unquoteFilename(splitLine[2]) - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, fmt.Errorf("failed to unquote new filename: %s", err) - } - - if latestFilename, alreadyRenamed := renameMap[newFilename]; alreadyRenamed { - delete(renameMap, newFilename) - newFilename = latestFilename - } - - renameMap[prevFilename] = newFilename - } - - if err := scanner.Err(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - stdout.Close() - - if err := cmd.Wait(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, newError(err, logWriter.AllOutput()) - } - - result := []RenamedFile{} - for prevFilename, newFilename := range renameMap { - result = append(result, RenamedFile{PreviousFilename: prevFilename, NewFilename: newFilename}) - } - - return result, nil -} diff --git a/internal/git/tree.go b/internal/git/tree.go index b14940ac4..4badf91e6 100644 --- a/internal/git/tree.go +++ b/internal/git/tree.go @@ -3,262 +3,64 @@ package git import ( "bufio" "context" - "fmt" "io" - "path" - "slices" "strings" - "time" - - "github.com/bearer/bearer/internal/util/file" ) -const blankID = "0000000000000000000000000000000000000000" - -type Tree struct { - Commit CommitIdentifier `json:"commit" yaml:"commit"` - Files []TreeFile `json:"files" yaml:"files"` -} - type TreeFile struct { Filename string `json:"filename" yaml:"filename"` SHA string `json:"sha" yaml:"sha"` } -func GetRoot(targetPath string) string { - dir := targetPath - if !file.IsDir(dir) { - dir = path.Dir(dir) - } - - command := logAndBuildCommand(context.TODO(), "rev-parse", "--show-toplevel") - command.Dir = dir - - output, err := command.Output() - if err != nil { - return "" - } - - path := strings.TrimSpace(string(output)) - if path == "" { - return "" - } - - canonicalPath, _ := file.CanonicalPath(path) - return canonicalPath -} - func HasUncommittedChanges(rootDir string) (bool, error) { output, err := captureCommandBasic( context.TODO(), rootDir, "status", - " --porcelain=v1", - " --no-renames", + "--porcelain=v1", + "--no-renames", ) - return strings.Count(output, "\n") > 0, err -} - -func GetTree(rootDir string) (*Tree, error) { - commit, err := getHeadCommitIdentifier(rootDir) - if err != nil { - return nil, fmt.Errorf("failed to get commit identifier: %s", err) - } - - files, err := ListTree(rootDir, commit.SHA) - if err != nil { - return nil, fmt.Errorf("failed to list tree: %s", err) - } - - return &Tree{Commit: *commit, Files: files}, nil + return strings.TrimSpace(output) != "", err } func ListTree(rootDir, commitSHA string) ([]TreeFile, error) { result := []TreeFile{} - cmd := logAndBuildCommand(context.TODO(), "ls-tree", "-r", "-z", commitSHA) - cmd.Dir = rootDir - - logWriter := &debugLogWriter{} - cmd.Stderr = logWriter - - stdout, err := cmd.StdoutPipe() - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - if err := cmd.Start(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - stdoutBuf := bufio.NewReader(stdout) - for { - metadata, err := stdoutBuf.ReadString('\t') - if err == io.EOF { - break - } - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - splitMeta := strings.Split(metadata[:len(metadata)-1], " ") - if len(splitMeta) != 3 { - continue - } - sha := splitMeta[2] - - filename, err := stdoutBuf.ReadString(0) - if err != nil && err != io.EOF { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - if len(filename) > 1 { - result = append(result, TreeFile{Filename: filename[:len(filename)-1], SHA: sha}) - } - } - - stdout.Close() - - if err := cmd.Wait(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, newError(err, logWriter.AllOutput()) - } - - return result, nil -} - -func getObjectIDsForRangeFiles(rootDir, firstCommitSHA, lastCommitSHA string, filenames []string) ([]string, error) { - firstCommitFileObjectIDs, err := getObjectIDsForFiles(rootDir, firstCommitSHA, filenames) - if err != nil { - return nil, err - } - - rangeUsedObjectIDs, err := getObjectIDsUsedByRange(rootDir, firstCommitSHA, lastCommitSHA) - if err != nil { - return nil, err - } - - ids := append(firstCommitFileObjectIDs, rangeUsedObjectIDs...) - slices.Sort(ids) - return slices.Compact(ids), nil -} - -// Returns all the object ids of files touched by the given range of commits. -func getObjectIDsUsedByRange(rootDir, firstCommitSHA, lastCommitSHA string) ([]string, error) { - result := []string{} - - cmd := logAndBuildCommand( + err := captureCommand( context.TODO(), - "log", - "--no-renames", - "--first-parent", - "--raw", - "--no-abbrev", - "--format=%H", - firstCommitSHA+".."+lastCommitSHA, + rootDir, + []string{"ls-tree", "-r", "-z", commitSHA}, + func(stdout io.Reader) error { + stdoutBuf := bufio.NewReader(stdout) + for { + metadata, err := stdoutBuf.ReadString('\t') + if err == io.EOF { + break + } + if err != nil { + return err + } + + splitMeta := strings.Split(metadata[:len(metadata)-1], " ") + if len(splitMeta) != 3 { + continue + } + sha := splitMeta[2] + + filename, err := stdoutBuf.ReadString(0) + if err != nil && err != io.EOF { + return err + } + + if len(filename) > 1 { + result = append(result, TreeFile{Filename: filename[:len(filename)-1], SHA: sha}) + } + } + + return nil + }, ) - cmd.Dir = rootDir - - logWriter := &debugLogWriter{} - cmd.Stderr = logWriter - - stdout, err := cmd.StdoutPipe() - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - if err := cmd.Start(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - line := scanner.Text() - if line == "" || line[0] != ':' { - continue - } - - splitLine := strings.Split(line, " ") - if splitLine[2] != blankID { - result = append(result, splitLine[2]) - } - if splitLine[3] != blankID { - result = append(result, splitLine[3]) - } - } - - if err := scanner.Err(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - stdout.Close() - - if err := cmd.Wait(); err != nil { - cmd.Cancel() //nolint:errcheck - return nil, newError(err, logWriter.AllOutput()) - } - - return result, nil -} - -// Get's the object ids of the given filenames in the specified commit. -// Also includes special git metadata files -func getObjectIDsForFiles(rootDir, commitSHA string, filenames []string) ([]string, error) { - wantedFilenames := make(map[string]struct{}) - for _, filename := range filenames { - wantedFilenames[filename] = struct{}{} - } - - treeFiles, err := ListTree(rootDir, commitSHA) - if err != nil { - return nil, err - } - - objectIDs := []string{} - - for _, treeFile := range treeFiles { - if _, ok := wantedFilenames[treeFile.Filename]; ok { - objectIDs = append(objectIDs, treeFile.SHA) - continue - } - - if slices.Contains(specialFiles, path.Base(treeFile.Filename)) { - objectIDs = append(objectIDs, treeFile.SHA) - } - } - - return objectIDs, nil -} - -func getHeadCommitIdentifier(rootDir string) (*CommitIdentifier, error) { - cmd := logAndBuildCommand(context.TODO(), "log", "-1", "--format=%H %cI") - cmd.Dir = rootDir - - logWriter := &debugLogWriter{} - cmd.Stderr = logWriter - - output, err := cmd.Output() - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, newError(err, logWriter.AllOutput()) - } - - splitOutput := strings.SplitN(strings.TrimSpace(string(output)), " ", 2) - - parsedTimestamp, err := time.Parse(time.RFC3339, splitOutput[1]) - if err != nil { - cmd.Cancel() //nolint:errcheck - return nil, err - } - - return &CommitIdentifier{SHA: splitOutput[0], Timestamp: parsedTimestamp.UTC()}, nil + return result, err } diff --git a/internal/report/output/output.go b/internal/report/output/output.go index bf9da13ca..f1b450d84 100644 --- a/internal/report/output/output.go +++ b/internal/report/output/output.go @@ -63,7 +63,7 @@ func GetData( if err = security.AddReportData(data, config, baseBranchFindings, report.HasFiles); err != nil { return nil, err } - err = saas.GetReport(data, config, gitContext, false) + err = saas.GetReport(data, config, gitContext, true) case flag.ReportPrivacy: err = privacy.AddReportData(data, config) case flag.ReportStats: