feat: add fail-on-severity flag

docs/_data/bearer_scan.yaml

@@ -45,6 +45,10 @@ options:
       default_value: '[]'
       usage: |
         Specify directories paths that contain .yaml files with external rules configuration
+    - name: fail-on-severity
+      default_value: critical,high,medium,low
+      usage: |
+        Specify which severities cause the report to fail. Works in conjunction with --exit-code.
     - name: force
       default_value: "false"
       usage: Disable the cache and runs the detections again

e2e/flags/.snapshots/TestMetadataFlags-help-scan

@@ -10,10 +10,11 @@ Examples:
 
 
 Report Flags
-  -f, --format string     Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
-      --output string     Specify the output path for the report.
-      --report string     Specify the type of report (security, privacy, dataflow). (default "security")
-      --severity string   Specify which severities are included in the report. (default "critical,high,medium,low,warning")
+      --fail-on-severity string   Specify which severities cause the report to fail. Works in conjunction with --exit-code. (default "critical,high,medium,low")
+  -f, --format string             Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+      --output string             Specify the output path for the report.
+      --report string             Specify the type of report (security, privacy, dataflow). (default "security")
+      --severity string           Specify which severities are included in the report. (default "critical,high,medium,low,warning")
 
 Rule Flags
       --disable-default-rules   Disables all default and built-in rules.

e2e/flags/.snapshots/TestMetadataFlags-scan-help

@@ -10,10 +10,11 @@ Examples:
 
 
 Report Flags
-  -f, --format string     Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
-      --output string     Specify the output path for the report.
-      --report string     Specify the type of report (security, privacy, dataflow). (default "security")
-      --severity string   Specify which severities are included in the report. (default "critical,high,medium,low,warning")
+      --fail-on-severity string   Specify which severities cause the report to fail. Works in conjunction with --exit-code. (default "critical,high,medium,low")
+  -f, --format string             Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+      --output string             Specify the output path for the report.
+      --report string             Specify the type of report (security, privacy, dataflow). (default "security")
+      --severity string           Specify which severities are included in the report. (default "critical,high,medium,low,warning")
 
 Rule Flags
       --disable-default-rules   Disables all default and built-in rules.

internal/flag/options.go

@@ -10,6 +10,9 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+
+	"github.com/bearer/bearer/internal/types"
+	"github.com/bearer/bearer/internal/util/set"
 )
 
 var ErrInvalidScannerReportCombination = errors.New("invalid scanner argument; privacy report requires sast scanner")
@@ -177,6 +180,20 @@ func getInteger(flag *Flag) int {
 	return viper.GetInt(flag.ConfigName)
 }
 
+func getSeverities(flag *Flag) set.Set[string] {
+	result := set.New[string]()
+
+	for _, value := range getStringSlice(flag) {
+		if !slices.Contains(types.Severities, value) {
+			return nil
+		}
+
+		result.Add(value)
+	}
+
+	return result
+}
+
 func (f *Flags) groups() []FlagGroup {
 	var groups []FlagGroup
 	// This order affects the usage message, so they are sorted by frequency of use.

internal/flag/report_flags.go

@@ -2,8 +2,11 @@ package flag
 
 import (
 	"errors"
+	"strings"
 
-	"github.com/bearer/bearer/internal/types"
+	globaltypes "github.com/bearer/bearer/internal/types"
+	"github.com/bearer/bearer/internal/util/set"
+	sliceutil "github.com/bearer/bearer/internal/util/slices"
 )
 
 var (
@@ -22,15 +25,16 @@ var (
 	ReportDetectors = "detectors" // nodoc: internal report type
 	ReportSaaS      = "saas"      // nodoc: internal report type
 	ReportStats     = "stats"     // nodoc: internal report type
-
-	DefaultSeverity = "critical,high,medium,low,warning"
 )
 
-var ErrInvalidFormatSecurity = errors.New("invalid format argument for security report; supported values: json, yaml, sarif, gitlab-sast, rdjson, html")
-var ErrInvalidFormatPrivacy = errors.New("invalid format argument for privacy report; supported values: csv, json, yaml, html")
-var ErrInvalidFormatDefault = errors.New("invalid format argument; supported values: json, yaml")
-var ErrInvalidReport = errors.New("invalid report argument; supported values: security, privacy")
-var ErrInvalidSeverity = errors.New("invalid severity argument; supported values: critical, high, medium, low, warning")
+var (
+	ErrInvalidFormatSecurity = errors.New("invalid format argument for security report; supported values: json, yaml, sarif, gitlab-sast, rdjson, html")
+	ErrInvalidFormatPrivacy  = errors.New("invalid format argument for privacy report; supported values: csv, json, yaml, html")
+	ErrInvalidFormatDefault  = errors.New("invalid format argument; supported values: json, yaml")
+	ErrInvalidReport         = errors.New("invalid report argument; supported values: security, privacy")
+	ErrInvalidSeverity       = errors.New("invalid severity argument; supported values: " + strings.Join(globaltypes.Severities, ", "))
+	ErrInvalidFailOnSeverity = errors.New("invalid fail-on-severity argument; supported values: " + strings.Join(globaltypes.Severities, ", "))
+)
 
 var (
 	FormatFlag = Flag{
@@ -55,9 +59,15 @@ var (
 	SeverityFlag = Flag{
 		Name:       "severity",
 		ConfigName: "report.severity",
-		Value:      DefaultSeverity,
+		Value:      strings.Join(globaltypes.Severities, ","),
 		Usage:      "Specify which severities are included in the report.",
 	}
+	FailOnSeverityFlag = Flag{
+		Name:       "fail-on-severity",
+		ConfigName: "report.fail-on-severity",
+		Value:      strings.Join(sliceutil.Except(globaltypes.Severities, globaltypes.LevelWarning), ","),
+		Usage:      "Specify which severities cause the report to fail. Works in conjunction with --exit-code.",
+	}
 	ExcludeFingerprintFlag = Flag{
 		Name:            "exclude-fingerprint",
 		ConfigName:      "report.exclude-fingerprint",
@@ -74,14 +84,16 @@ type ReportFlagGroup struct {
 	Report             *Flag
 	Output             *Flag
 	Severity           *Flag
+	FailOnSeverity     *Flag
 	ExcludeFingerprint *Flag
 }
 
 type ReportOptions struct {
 	Format             string          `mapstructure:"format" json:"format" yaml:"format"`
 	Report             string          `mapstructure:"report" json:"report" yaml:"report"`
 	Output             string          `mapstructure:"output" json:"output" yaml:"output"`
-	Severity           map[string]bool `mapstructure:"severity" json:"severity" yaml:"severity"`
+	Severity           set.Set[string] `mapstructure:"severity" json:"severity" yaml:"severity"`
+	FailOnSeverity     set.Set[string] `mapstructure:"fail-on-severity" json:"fail-on-severity" yaml:"fail-on-severity"`
 	ExcludeFingerprint map[string]bool `mapstructure:"exclude_fingerprints" json:"exclude_fingerprints" yaml:"exclude_fingerprints"`
 }
 
@@ -91,6 +103,7 @@ func NewReportFlagGroup() *ReportFlagGroup {
 		Report:             &ReportFlag,
 		Output:             &OutputFlag,
 		Severity:           &SeverityFlag,
+		FailOnSeverity:     &FailOnSeverityFlag,
 		ExcludeFingerprint: &ExcludeFingerprintFlag,
 	}
 }
@@ -105,6 +118,7 @@ func (f *ReportFlagGroup) Flags() []*Flag {
 		f.Report,
 		f.Output,
 		f.Severity,
+		f.FailOnSeverity,
 		f.ExcludeFingerprint,
 	}
 }
@@ -147,24 +161,13 @@ func (f *ReportFlagGroup) ToOptions() (ReportOptions, error) {
 		return ReportOptions{}, invalidFormat
 	}
 
-	severity := getStringSlice(f.Severity)
-	severityMapping := make(map[string]bool)
-
-	for _, severityLevel := range severity {
-		switch severityLevel {
-		case types.LevelCritical:
-			severityMapping[types.LevelCritical] = true
-		case types.LevelHigh:
-			severityMapping[types.LevelHigh] = true
-		case types.LevelMedium:
-			severityMapping[types.LevelMedium] = true
-		case types.LevelLow:
-			severityMapping[types.LevelLow] = true
-		case types.LevelWarning:
-			severityMapping[types.LevelWarning] = true
-		default:
-			return ReportOptions{}, ErrInvalidSeverity
-		}
+	severity := getSeverities(f.Severity)
+	if severity == nil {
+		return ReportOptions{}, ErrInvalidSeverity
+	}
+	failOnSeverity := getSeverities(f.FailOnSeverity)
+	if failOnSeverity == nil {
+		return ReportOptions{}, ErrInvalidFailOnSeverity
 	}
 
 	// turn string slice into map for ease of access
@@ -178,7 +181,8 @@ func (f *ReportFlagGroup) ToOptions() (ReportOptions, error) {
 		Format:             format,
 		Report:             report,
 		Output:             getString(f.Output),
-		Severity:           severityMapping,
+		Severity:           severity,
+		FailOnSeverity:     failOnSeverity,
 		ExcludeFingerprint: excludeFingerprintsMapping,
 	}, nil
 }

internal/report/output/security/security.go

@@ -41,13 +41,6 @@ var severityColorFns = map[string]func(x ...interface{}) string{
 	globaltypes.LevelLow:      color.New(color.FgBlue).SprintFunc(),
 	globaltypes.LevelWarning:  color.New(color.FgCyan).SprintFunc(),
 }
-var orderedSeverityLevels = []string{
-	globaltypes.LevelCritical,
-	globaltypes.LevelHigh,
-	globaltypes.LevelMedium,
-	globaltypes.LevelLow,
-	globaltypes.LevelWarning,
-}
 
 type Findings = map[string][]types.Finding
 type IgnoredFindings = map[string][]types.IgnoredFinding
@@ -89,11 +82,11 @@ func AddReportData(
 		output.StdErrLog("Evaluating rules")
 	}
 
-	builtInFingerprints, err := evaluateRules(summaryFindings, ignoredSummaryFindings, config.BuiltInRules, config, dataflow, baseBranchFindings, true)
+	builtInFingerprints, builtInFailed, err := evaluateRules(summaryFindings, ignoredSummaryFindings, config.BuiltInRules, config, dataflow, baseBranchFindings, true)
 	if err != nil {
 		return err
 	}
-	fingerprints, err := evaluateRules(summaryFindings, ignoredSummaryFindings, config.Rules, config, dataflow, baseBranchFindings, false)
+	fingerprints, failed, err := evaluateRules(summaryFindings, ignoredSummaryFindings, config.Rules, config, dataflow, baseBranchFindings, false)
 	if err != nil {
 		return err
 	}
@@ -109,17 +102,9 @@ func AddReportData(
 		)
 	}
 
-	// fail the report if we have failures above the severity threshold
-	reportFailed := false
-	for severityLevel, findings := range summaryFindings {
-		if severityLevel != globaltypes.LevelWarning && len(findings) != 0 {
-			reportFailed = true
-		}
-	}
-
 	reportData.FindingsBySeverity = summaryFindings
 	reportData.IgnoredFindingsBySeverity = ignoredSummaryFindings
-	reportData.ReportFailed = reportFailed
+	reportData.ReportFailed = builtInFailed || failed
 	return nil
 }
 
@@ -131,7 +116,7 @@ func evaluateRules(
 	dataflow *outputtypes.DataFlow,
 	baseBranchFindings *basebranchfindings.Findings,
 	builtIn bool,
-) ([]string, error) {
+) ([]string, bool, error) {
 	outputFindings := map[string][]types.Finding{}
 	ignoredOutputFindings := map[string][]types.IgnoredFinding{}
 
@@ -141,6 +126,7 @@ func evaluateRules(
 	}
 
 	var fingerprints []string
+	failed := false
 
 	for _, rule := range maputil.ToSortedSlice(rules) {
 		if !builtIn {
@@ -166,19 +152,19 @@ func evaluateRules(
 			// TODO: perf question: can we do this once?
 			policy.Modules.ToRegoModules())
 		if err != nil {
-			return fingerprints, err
+			return fingerprints, false, err
 		}
 
 		if len(rs) > 0 {
 			jsonRes, err := json.Marshal(rs)
 			if err != nil {
-				return fingerprints, err
+				return fingerprints, false, err
 			}
 
 			var results map[string][]Output
 			err = json.Unmarshal(jsonRes, &results)
 			if err != nil {
-				return fingerprints, err
+				return fingerprints, false, err
 			}
 
 			ruleSummary := &types.Rule{
@@ -236,22 +222,26 @@ func evaluateRules(
 				severityMeta := CalculateSeverity(finding.CategoryGroups, rule.GetSeverity(), output.IsLocal != nil && *output.IsLocal)
 				severity := severityMeta.DisplaySeverity
 
-				if config.Report.Severity[severity] {
+				if config.Report.Severity.Has(severity) {
 					finding.SeverityMeta = severityMeta
 					if ignored {
 						ignoredOutputFindings[severity] = append(ignoredOutputFindings[severity], types.IgnoredFinding{Finding: finding, IgnoreMeta: ignoredFingerprint})
 					} else {
 						outputFindings[severity] = append(outputFindings[severity], finding)
 					}
 				}
+
+				if config.Report.FailOnSeverity.Has(severity) && !ignored {
+					failed = true
+				}
 			}
 		}
 	}
 
 	sortFindingsBySeverity(summaryFindings, outputFindings)
 	sortFindingsBySeverity(ignoredSummaryFindings, ignoredOutputFindings)
 
-	return fingerprints, nil
+	return fingerprints, failed, nil
 }
 
 func sortFindingsBySeverity[F types.GenericFinding](findingsBySeverity map[string][]F, outputFindings map[string][]F) {
@@ -400,7 +390,7 @@ func BuildReportString(reportData *outputtypes.ReportData, config settings.Confi
 		globaltypes.LevelWarning:  make(map[string]bool),
 	}
 
-	for _, severityLevel := range orderedSeverityLevels {
+	for _, severityLevel := range globaltypes.Severities {
 		for _, failure := range reportData.FindingsBySeverity[severityLevel] {
 			for i := 0; i < len(failure.CWEIDs); i++ {
 				failures[severityLevel]["CWE-"+failure.CWEIDs[i]] = true
@@ -661,7 +651,7 @@ func checkAndWriteFailureSummaryToString(
 	findings Findings,
 	ruleCount int,
 	failures map[string]map[string]bool,
-	severityForFailure map[string]bool,
+	reportedSeverity set.Set[string],
 ) bool {
 	reportStr.WriteString("\n=====================================")
 
@@ -672,7 +662,7 @@ func checkAndWriteFailureSummaryToString(
 	// give summary including counts
 	failureCount := 0
 	warningCount := 0
-	for _, severityLevel := range maps.Keys(severityForFailure) {
+	for _, severityLevel := range globaltypes.Severities {
 		if severityLevel == globaltypes.LevelWarning {
 			warningCount += len(findings[severityLevel])
 			continue
@@ -687,8 +677,8 @@ func checkAndWriteFailureSummaryToString(
 	reportStr.WriteString("\n\n")
 	reportStr.WriteString(color.RedString(fmt.Sprint(ruleCount) + " checks, " + fmt.Sprint(failureCount+warningCount) + " findings\n"))
 
-	for _, severityLevel := range orderedSeverityLevels {
-		if !severityForFailure[severityLevel] {
+	for _, severityLevel := range globaltypes.Severities {
+		if !reportedSeverity.Has(severityLevel) {
 			continue
 		}
 		reportStr.WriteString("\n" + formatSeverity(severityLevel) + fmt.Sprint(len(findings[severityLevel])))
@@ -756,7 +746,7 @@ func removeDuplicates[F types.GenericFinding](data map[string][]F) map[string][]
 	reportedDetections := set.Set[key]{}
 
 	// filter duplicates
-	for _, severity := range orderedSeverityLevels {
+	for _, severity := range globaltypes.Severities {
 		findingsSlice, hasSeverity := data[severity]
 		if !hasSeverity {
 			continue