feat: add fail-on-severity flag (#1351)

* feat: add fail-on-severity flag

* refactor: use single source of truth for severity levels

* test: update help snapshot

* docs: regenerate scan flags documentation

* test: update remaining snapshots

docs/_data/bearer_scan.yaml

@@ -45,6 +45,10 @@ options:
       default_value: '[]'
       usage: |
         Specify directories paths that contain .yaml files with external rules configuration
+    - name: fail-on-severity
+      default_value: critical,high,medium,low
+      usage: |
+        Specify which severities cause the report to fail. Works in conjunction with --exit-code.
     - name: force
       default_value: "false"
       usage: Disable the cache and runs the detections again

e2e/flags/.snapshots/TestInitCommand

@@ -1,6 +1,7 @@
 disable-version-check: false
 log-level: info
 report:
+    fail-on-severity: critical,high,medium,low
     format: ""
     no-color: false
     output: ""

e2e/flags/.snapshots/TestMetadataFlags-help-scan

@@ -10,10 +10,11 @@ Examples:
 
 
 Report Flags
-  -f, --format string     Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
-      --output string     Specify the output path for the report.
-      --report string     Specify the type of report (security, privacy, dataflow). (default "security")
-      --severity string   Specify which severities are included in the report. (default "critical,high,medium,low,warning")
+      --fail-on-severity string   Specify which severities cause the report to fail. Works in conjunction with --exit-code. (default "critical,high,medium,low")
+  -f, --format string             Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+      --output string             Specify the output path for the report.
+      --report string             Specify the type of report (security, privacy, dataflow). (default "security")
+      --severity string           Specify which severities are included in the report. (default "critical,high,medium,low,warning")
 
 Rule Flags
       --disable-default-rules   Disables all default and built-in rules.

e2e/flags/.snapshots/TestMetadataFlags-scan-help

@@ -10,10 +10,11 @@ Examples:
 
 
 Report Flags
-  -f, --format string     Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
-      --output string     Specify the output path for the report.
-      --report string     Specify the type of report (security, privacy, dataflow). (default "security")
-      --severity string   Specify which severities are included in the report. (default "critical,high,medium,low,warning")
+      --fail-on-severity string   Specify which severities cause the report to fail. Works in conjunction with --exit-code. (default "critical,high,medium,low")
+  -f, --format string             Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+      --output string             Specify the output path for the report.
+      --report string             Specify the type of report (security, privacy, dataflow). (default "security")
+      --severity string           Specify which severities are included in the report. (default "critical,high,medium,low,warning")
 
 Rule Flags
       --disable-default-rules   Disables all default and built-in rules.

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-context-flag

@@ -11,10 +11,11 @@ Examples:
 
 
 Report Flags
-  -f, --format string     Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
-      --output string     Specify the output path for the report.
-      --report string     Specify the type of report (security, privacy, dataflow). (default "security")
-      --severity string   Specify which severities are included in the report. (default "critical,high,medium,low,warning")
+      --fail-on-severity string   Specify which severities cause the report to fail. Works in conjunction with --exit-code. (default "critical,high,medium,low")
+  -f, --format string             Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+      --output string             Specify the output path for the report.
+      --report string             Specify the type of report (security, privacy, dataflow). (default "security")
+      --severity string           Specify which severities are included in the report. (default "critical,high,medium,low,warning")
 
 Rule Flags
       --disable-default-rules   Disables all default and built-in rules.

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-privacy

@@ -11,10 +11,11 @@ Examples:
 
 
 Report Flags
-  -f, --format string     Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
-      --output string     Specify the output path for the report.
-      --report string     Specify the type of report (security, privacy, dataflow). (default "security")
-      --severity string   Specify which severities are included in the report. (default "critical,high,medium,low,warning")
+      --fail-on-severity string   Specify which severities cause the report to fail. Works in conjunction with --exit-code. (default "critical,high,medium,low")
+  -f, --format string             Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+      --output string             Specify the output path for the report.
+      --report string             Specify the type of report (security, privacy, dataflow). (default "security")
+      --severity string           Specify which severities are included in the report. (default "critical,high,medium,low,warning")
 
 Rule Flags
       --disable-default-rules   Disables all default and built-in rules.

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-format-flag-security

@@ -11,10 +11,11 @@ Examples:
 
 
 Report Flags
-  -f, --format string     Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
-      --output string     Specify the output path for the report.
-      --report string     Specify the type of report (security, privacy, dataflow). (default "security")
-      --severity string   Specify which severities are included in the report. (default "critical,high,medium,low,warning")
+      --fail-on-severity string   Specify which severities cause the report to fail. Works in conjunction with --exit-code. (default "critical,high,medium,low")
+  -f, --format string             Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+      --output string             Specify the output path for the report.
+      --report string             Specify the type of report (security, privacy, dataflow). (default "security")
+      --severity string           Specify which severities are included in the report. (default "critical,high,medium,low,warning")
 
 Rule Flags
       --disable-default-rules   Disables all default and built-in rules.

e2e/flags/.snapshots/TestReportFlagsShouldFail-invalid-report-flag

@@ -11,10 +11,11 @@ Examples:
 
 
 Report Flags
-  -f, --format string     Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
-      --output string     Specify the output path for the report.
-      --report string     Specify the type of report (security, privacy, dataflow). (default "security")
-      --severity string   Specify which severities are included in the report. (default "critical,high,medium,low,warning")
+      --fail-on-severity string   Specify which severities cause the report to fail. Works in conjunction with --exit-code. (default "critical,high,medium,low")
+  -f, --format string             Specify report format (json, yaml, sarif, gitlab-sast, rdjson, html)
+      --output string             Specify the output path for the report.
+      --report string             Specify the type of report (security, privacy, dataflow). (default "security")
+      --severity string           Specify which severities are included in the report. (default "critical,high,medium,low,warning")
 
 Rule Flags
       --disable-default-rules   Disables all default and built-in rules.

internal/flag/options.go

@@ -10,6 +10,9 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+
+	"github.com/bearer/bearer/internal/types"
+	"github.com/bearer/bearer/internal/util/set"
 )
 
 var ErrInvalidScannerReportCombination = errors.New("invalid scanner argument; privacy report requires sast scanner")
@@ -177,6 +180,20 @@ func getInteger(flag *Flag) int {
 	return viper.GetInt(flag.ConfigName)
 }
 
+func getSeverities(flag *Flag) set.Set[string] {
+	result := set.New[string]()
+
+	for _, value := range getStringSlice(flag) {
+		if !slices.Contains(types.Severities, value) {
+			return nil
+		}
+
+		result.Add(value)
+	}
+
+	return result
+}
+
 func (f *Flags) groups() []FlagGroup {
 	var groups []FlagGroup
 	// This order affects the usage message, so they are sorted by frequency of use.

internal/flag/report_flags.go

@@ -2,8 +2,11 @@ package flag
 
 import (
 	"errors"
+	"strings"
 
-	"github.com/bearer/bearer/internal/types"
+	globaltypes "github.com/bearer/bearer/internal/types"
+	"github.com/bearer/bearer/internal/util/set"
+	sliceutil "github.com/bearer/bearer/internal/util/slices"
 )
 
 var (
@@ -22,15 +25,16 @@ var (
 	ReportDetectors = "detectors" // nodoc: internal report type
 	ReportSaaS      = "saas"      // nodoc: internal report type
 	ReportStats     = "stats"     // nodoc: internal report type
-
-	DefaultSeverity = "critical,high,medium,low,warning"
 )
 
-var ErrInvalidFormatSecurity = errors.New("invalid format argument for security report; supported values: json, yaml, sarif, gitlab-sast, rdjson, html")
-var ErrInvalidFormatPrivacy = errors.New("invalid format argument for privacy report; supported values: csv, json, yaml, html")
-var ErrInvalidFormatDefault = errors.New("invalid format argument; supported values: json, yaml")
-var ErrInvalidReport = errors.New("invalid report argument; supported values: security, privacy")
-var ErrInvalidSeverity = errors.New("invalid severity argument; supported values: critical, high, medium, low, warning")
+var (
+	ErrInvalidFormatSecurity = errors.New("invalid format argument for security report; supported values: json, yaml, sarif, gitlab-sast, rdjson, html")
+	ErrInvalidFormatPrivacy  = errors.New("invalid format argument for privacy report; supported values: csv, json, yaml, html")
+	ErrInvalidFormatDefault  = errors.New("invalid format argument; supported values: json, yaml")
+	ErrInvalidReport         = errors.New("invalid report argument; supported values: security, privacy")
+	ErrInvalidSeverity       = errors.New("invalid severity argument; supported values: " + strings.Join(globaltypes.Severities, ", "))
+	ErrInvalidFailOnSeverity = errors.New("invalid fail-on-severity argument; supported values: " + strings.Join(globaltypes.Severities, ", "))
+)
 
 var (
 	FormatFlag = Flag{
@@ -55,9 +59,15 @@ var (
 	SeverityFlag = Flag{
 		Name:       "severity",
 		ConfigName: "report.severity",
-		Value:      DefaultSeverity,
+		Value:      strings.Join(globaltypes.Severities, ","),
 		Usage:      "Specify which severities are included in the report.",
 	}
+	FailOnSeverityFlag = Flag{
+		Name:       "fail-on-severity",
+		ConfigName: "report.fail-on-severity",
+		Value:      strings.Join(sliceutil.Except(globaltypes.Severities, globaltypes.LevelWarning), ","),
+		Usage:      "Specify which severities cause the report to fail. Works in conjunction with --exit-code.",
+	}
 	ExcludeFingerprintFlag = Flag{
 		Name:            "exclude-fingerprint",
 		ConfigName:      "report.exclude-fingerprint",
@@ -74,14 +84,16 @@ type ReportFlagGroup struct {
 	Report             *Flag
 	Output             *Flag
 	Severity           *Flag
+	FailOnSeverity     *Flag
 	ExcludeFingerprint *Flag
 }
 
 type ReportOptions struct {
 	Format             string          `mapstructure:"format" json:"format" yaml:"format"`
 	Report             string          `mapstructure:"report" json:"report" yaml:"report"`
 	Output             string          `mapstructure:"output" json:"output" yaml:"output"`
-	Severity           map[string]bool `mapstructure:"severity" json:"severity" yaml:"severity"`
+	Severity           set.Set[string] `mapstructure:"severity" json:"severity" yaml:"severity"`
+	FailOnSeverity     set.Set[string] `mapstructure:"fail-on-severity" json:"fail-on-severity" yaml:"fail-on-severity"`
 	ExcludeFingerprint map[string]bool `mapstructure:"exclude_fingerprints" json:"exclude_fingerprints" yaml:"exclude_fingerprints"`
 }
 
@@ -91,6 +103,7 @@ func NewReportFlagGroup() *ReportFlagGroup {
 		Report:             &ReportFlag,
 		Output:             &OutputFlag,
 		Severity:           &SeverityFlag,
+		FailOnSeverity:     &FailOnSeverityFlag,
 		ExcludeFingerprint: &ExcludeFingerprintFlag,
 	}
 }
@@ -105,6 +118,7 @@ func (f *ReportFlagGroup) Flags() []*Flag {
 		f.Report,
 		f.Output,
 		f.Severity,
+		f.FailOnSeverity,
 		f.ExcludeFingerprint,
 	}
 }
@@ -147,24 +161,13 @@ func (f *ReportFlagGroup) ToOptions() (ReportOptions, error) {
 		return ReportOptions{}, invalidFormat
 	}
 
-	severity := getStringSlice(f.Severity)
-	severityMapping := make(map[string]bool)
-
-	for _, severityLevel := range severity {
-		switch severityLevel {
-		case types.LevelCritical:
-			severityMapping[types.LevelCritical] = true
-		case types.LevelHigh:
-			severityMapping[types.LevelHigh] = true
-		case types.LevelMedium:
-			severityMapping[types.LevelMedium] = true
-		case types.LevelLow:
-			severityMapping[types.LevelLow] = true
-		case types.LevelWarning:
-			severityMapping[types.LevelWarning] = true
-		default:
-			return ReportOptions{}, ErrInvalidSeverity
-		}
+	severity := getSeverities(f.Severity)
+	if severity == nil {
+		return ReportOptions{}, ErrInvalidSeverity
+	}
+	failOnSeverity := getSeverities(f.FailOnSeverity)
+	if failOnSeverity == nil {
+		return ReportOptions{}, ErrInvalidFailOnSeverity
 	}
 
 	// turn string slice into map for ease of access
@@ -178,7 +181,8 @@ func (f *ReportFlagGroup) ToOptions() (ReportOptions, error) {
 		Format:             format,
 		Report:             report,
 		Output:             getString(f.Output),
-		Severity:           severityMapping,
+		Severity:           severity,
+		FailOnSeverity:     failOnSeverity,
 		ExcludeFingerprint: excludeFingerprintsMapping,
 	}, nil
 }