fix: use git cli for diff (#1409)

* fix: use git cli for diff

* fix: load git context info centrally

* fix: fixes from testing

* docs: update diff docs

* fix: handling of zero line case

* fix: more fixes from testing

* fix: prevent hang if error starting worker

* docs: update following flag rename

docs/_data/bearer_scan.yaml

@@ -4,6 +4,10 @@ usage: bearer scan [flags] <path>
 options:
   - name: api-key
     usage: Use your Bearer API Key to send the report to Bearer.
+  - name: branch
+    usage: The name of the branch being scanned.
+  - name: commit
+    usage: The hash of the commit being scanned.
   - name: config-file
     default_value: bearer.yml
     usage: Load configuration from the specified path.
@@ -19,6 +23,16 @@ options:
   - name: debug-profile
     default_value: "false"
     usage: Generate profiling data for debugging
+  - name: default-branch
+    usage: The name of the default branch.
+  - name: diff
+    default_value: "false"
+    usage: |
+      Only report differences in findings relative to a base branch.
+  - name: diff-base-branch
+    usage: The name of the base branch to use for diff scanning.
+  - name: diff-base-commit
+    usage: The hash of the base commit to use for diff scanning.
   - name: disable-default-rules
     default_value: "false"
     usage: Disables all default and built-in rules.
@@ -56,6 +70,13 @@ options:
     shorthand: f
     usage: |
       Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+  - name: github-api-url
+    usage: A non-standard URL to use for the Github API
+  - name: github-repository
+    usage: |
+      The owner and name of the repository on Github. eg. Bearer/bearer
+  - name: github-token
+    usage: An access token for the Github API.
   - name: help
     shorthand: h
     default_value: "false"
@@ -97,6 +118,8 @@ options:
   - name: report
     default_value: security
     usage: Specify the type of report (security, privacy, dataflow).
+  - name: repository-url
+    usage: The remote URL of the repository.
   - name: scanner
     default_value: "[sast]"
     usage: |

docs/_data/examples/ci/gitlab/diff-reviewdog.yaml

@@ -1,10 +1,7 @@
 bearer_mr:
-  variables:
-    DIFF_BASE_BRANCH: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
-    DIFF_BASE_COMMIT: $CI_MERGE_REQUEST_DIFF_BASE_SHA
   script:
     - curl -sfL https://raw.githubusercontent.com/Bearer/bearer/main/contrib/install.sh | sh -s -- -b /usr/local/bin
     - curl -sfL https://raw.githubusercontent.com/reviewdog/reviewdog/master/install.sh | sh -s -- -b /usr/local/bin
-    - bearer scan . --format=rdjson --output=rd.json || export BEARER_EXIT=$?
+    - bearer scan . --diff --format=rdjson --output=rd.json || export BEARER_EXIT=$?
     - cat rd.json | reviewdog -f=rdjson -reporter=gitlab-mr-discussion
     - exit $BEARER_EXIT

docs/_data/examples/ci/gitlab/diff.yaml

@@ -2,7 +2,4 @@ bearer_mr:
   image:
     name: bearer/bearer
     entrypoint: [""]
-  variables:
-    DIFF_BASE_BRANCH: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME
-    DIFF_BASE_COMMIT: $CI_MERGE_REQUEST_DIFF_BASE_SHA
-  script: bearer scan .
+  script: bearer scan --diff .

docs/guides/configure-scan.md

@@ -33,12 +33,13 @@ When scanning a Git repository, you can choose to only report new findings that
 have been introduced, relative to a base branch. Any findings that already
 existed in the base branch will not be reported.
 
-Use the `DIFF_BASE_BRANCH` environment variable to enable differential scanning,
-and to specify the base branch to use for comparison.
+Use the `--diff` flag to enable differential scanning. The repository's default
+branch will be used as the base branch for comparison. You can override this by
+setting the `BEARER_DIFF_BASE_BRANCH` environment variable.
 
 ```bash
 git checkout my-feature
-DIFF_BASE_BRANCH=main bearer scan .
+BEARER_DIFF_BASE_BRANCH=base-branch bearer scan --diff .
 ```
 
 If the base branch is not available in the git repository, it's head will be

docs/guides/gitlab.md

@@ -31,8 +31,8 @@ These changes set the format to `gitlab-sast` and write an artifact that GitLab
 ### Gitlab Merge Request Diff
 
 When Bearer CLI is being used to check a merge request, you can tell the Bearer
-CLI to only report findings introduced within the merge request by setting the
-`DIFF_BASE_BRANCH` variable.
+CLI to only report findings introduced within the merge request by adding the
+`--diff` flag.
 
 {% yamlExample "ci/gitlab/diff" %}
 

e2e/flags/.snapshots/TestMetadataFlags-help-scan

@@ -24,6 +24,7 @@ Rule Flags
 Scan Flags
       --context string                       Expand context of schema classification e.g., --context=health, to include data types particular to health
       --data-subject-mapping string          Override default data subject mapping by providing a path to a custom mapping JSON file
+      --diff                                 Only report differences in findings relative to a base branch.
       --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default true)
       --domain-resolution-timeout duration   Set timeout when attempting to resolve detected domains during classification, e.g. --domain-resolution-timeout=3s (default 3s)
       --exit-code int                        Force a given exit code for the scan command. Set this to 0 (success) to always return a success exit code despite any findings from the scan. (default -1)

e2e/flags/.snapshots/TestMetadataFlags-scan-help

@@ -24,6 +24,7 @@ Rule Flags
 Scan Flags
       --context string                       Expand context of schema classification e.g., --context=health, to include data types particular to health
       --data-subject-mapping string          Override default data subject mapping by providing a path to a custom mapping JSON file
+      --diff                                 Only report differences in findings relative to a base branch.
       --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default true)
       --domain-resolution-timeout duration   Set timeout when attempting to resolve detected domains during classification, e.g. --domain-resolution-timeout=3s (default 3s)
       --exit-code int                        Force a given exit code for the scan command. Set this to 0 (success) to always return a success exit code despite any findings from the scan. (default -1)

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-context-flag

@@ -25,6 +25,7 @@ Rule Flags
 Scan Flags
       --context string                       Expand context of schema classification e.g., --context=health, to include data types particular to health
       --data-subject-mapping string          Override default data subject mapping by providing a path to a custom mapping JSON file
+      --diff                                 Only report differences in findings relative to a base branch.
       --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default true)
       --domain-resolution-timeout duration   Set timeout when attempting to resolve detected domains during classification, e.g. --domain-resolution-timeout=3s (default 3s)
       --exit-code int                        Force a given exit code for the scan command. Set this to 0 (success) to always return a success exit code despite any findings from the scan. (default -1)

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-privacy

@@ -25,6 +25,7 @@ Rule Flags
 Scan Flags
       --context string                       Expand context of schema classification e.g., --context=health, to include data types particular to health
       --data-subject-mapping string          Override default data subject mapping by providing a path to a custom mapping JSON file
+      --diff                                 Only report differences in findings relative to a base branch.
       --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default true)
       --domain-resolution-timeout duration   Set timeout when attempting to resolve detected domains during classification, e.g. --domain-resolution-timeout=3s (default 3s)
       --exit-code int                        Force a given exit code for the scan command. Set this to 0 (success) to always return a success exit code despite any findings from the scan. (default -1)

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-security

@@ -25,6 +25,7 @@ Rule Flags
 Scan Flags
       --context string                       Expand context of schema classification e.g., --context=health, to include data types particular to health
       --data-subject-mapping string          Override default data subject mapping by providing a path to a custom mapping JSON file
+      --diff                                 Only report differences in findings relative to a base branch.
       --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default true)
       --domain-resolution-timeout duration   Set timeout when attempting to resolve detected domains during classification, e.g. --domain-resolution-timeout=3s (default 3s)
       --exit-code int                        Force a given exit code for the scan command. Set this to 0 (success) to always return a success exit code despite any findings from the scan. (default -1)

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-report-flag

@@ -25,6 +25,7 @@ Rule Flags
 Scan Flags
       --context string                       Expand context of schema classification e.g., --context=health, to include data types particular to health
       --data-subject-mapping string          Override default data subject mapping by providing a path to a custom mapping JSON file
+      --diff                                 Only report differences in findings relative to a base branch.
       --disable-domain-resolution            Do not attempt to resolve detected domains during classification (default true)
       --domain-resolution-timeout duration   Set timeout when attempting to resolve detected domains during classification, e.g. --domain-resolution-timeout=3s (default 3s)
       --exit-code int                        Force a given exit code for the scan command. Set this to 0 (success) to always return a success exit code despite any findings from the scan. (default -1)

go.mod

@@ -11,7 +11,6 @@ require (
 	github.com/gertd/go-pluralize v0.2.1
 	github.com/gitsight/go-vcsurl v1.0.1
 	github.com/go-enry/go-enry/v2 v2.8.4
-	github.com/go-git/go-git/v5 v5.10.0
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/uuid v1.4.0
 	github.com/hhatto/gocloc v0.5.2
@@ -69,63 +68,46 @@ require (
 )
 
 require (
-	dario.cat/mergo v1.0.0 // indirect
 	github.com/BobuSumisu/aho-corasick v1.0.3 // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
-	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cloudflare/circl v1.3.3 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
-	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-git/go-billy/v5 v5.5.0 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-logr/logr v1.3.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
 	github.com/sagikazarmark/locafero v0.3.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
-	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opentelemetry.io/otel v1.19.0 // indirect
 	go.opentelemetry.io/otel/metric v1.19.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.19.0 // indirect
 	go.opentelemetry.io/otel/trace v1.19.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.15.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
-	gopkg.in/warnings.v0 v0.1.2 // indirect
 )
 
 require (