Skip to content

Commit

Permalink
feat: add fail-on-severity flag
Browse files Browse the repository at this point in the history
  • Loading branch information
@didroe
didroe committed Oct 24, 2023
1 parent 6c0a0ae commit 0557383
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 94 deletions.
17 changes: 17 additions & 0 deletions internal/flag/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"github.com/spf13/viper"

"github.com/bearer/bearer/internal/types"
"github.com/bearer/bearer/internal/util/set"
)

var ErrInvalidScannerReportCombination = errors.New("invalid scanner argument; privacy report requires sast scanner")
Expand Down Expand Up @@ -177,6 +180,20 @@ func getInteger(flag *Flag) int {
return viper.GetInt(flag.ConfigName)
}

func getSeverities(flag *Flag) set.Set[string] {
result := set.New[string]()

for _, value := range getStringSlice(flag) {
if !slices.Contains(types.Severities, value) {
return nil
}

result.Add(value)
}

return result
}

func (f *Flags) groups() []FlagGroup {
var groups []FlagGroup
// This order affects the usage message, so they are sorted by frequency of use.
Expand Down
55 changes: 29 additions & 26 deletions internal/flag/report_flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package flag
import (
"errors"

"github.com/bearer/bearer/internal/types"
"github.com/bearer/bearer/internal/util/set"
)

var (
Expand All @@ -26,11 +26,14 @@ var (
DefaultSeverity = "critical,high,medium,low,warning"
)

var ErrInvalidFormatSecurity = errors.New("invalid format argument for security report; supported values: json, yaml, sarif, gitlab-sast, rdjson, html")
var ErrInvalidFormatPrivacy = errors.New("invalid format argument for privacy report; supported values: csv, json, yaml, html")
var ErrInvalidFormatDefault = errors.New("invalid format argument; supported values: json, yaml")
var ErrInvalidReport = errors.New("invalid report argument; supported values: security, privacy")
var ErrInvalidSeverity = errors.New("invalid severity argument; supported values: critical, high, medium, low, warning")
var (
ErrInvalidFormatSecurity = errors.New("invalid format argument for security report; supported values: json, yaml, sarif, gitlab-sast, rdjson, html")
ErrInvalidFormatPrivacy = errors.New("invalid format argument for privacy report; supported values: csv, json, yaml, html")
ErrInvalidFormatDefault = errors.New("invalid format argument; supported values: json, yaml")
ErrInvalidReport = errors.New("invalid report argument; supported values: security, privacy")
ErrInvalidSeverity = errors.New("invalid severity argument; supported values: critical, high, medium, low, warning")
ErrInvalidFailOnSeverity = errors.New("invalid fail-on-severity argument; supported values: critical, high, medium, low, warning")
)

var (
FormatFlag = Flag{
Expand Down Expand Up @@ -58,6 +61,12 @@ var (
Value: DefaultSeverity,
Usage: "Specify which severities are included in the report.",
}
FailOnSeverityFlag = Flag{
Name: "fail-on-severity",
ConfigName: "report.fail-on-severity",
Value: "critical,high,medium,low",
Usage: "Specify which severities cause the report to fail. Works in conjunction with --exit-code.",
}
ExcludeFingerprintFlag = Flag{
Name: "exclude-fingerprint",
ConfigName: "report.exclude-fingerprint",
Expand All @@ -74,14 +83,16 @@ type ReportFlagGroup struct {
Report *Flag
Output *Flag
Severity *Flag
FailOnSeverity *Flag
ExcludeFingerprint *Flag
}

type ReportOptions struct {
Format string `mapstructure:"format" json:"format" yaml:"format"`
Report string `mapstructure:"report" json:"report" yaml:"report"`
Output string `mapstructure:"output" json:"output" yaml:"output"`
Severity map[string]bool `mapstructure:"severity" json:"severity" yaml:"severity"`
Severity set.Set[string] `mapstructure:"severity" json:"severity" yaml:"severity"`
FailOnSeverity set.Set[string] `mapstructure:"fail-on-severity" json:"fail-on-severity" yaml:"fail-on-severity"`
ExcludeFingerprint map[string]bool `mapstructure:"exclude_fingerprints" json:"exclude_fingerprints" yaml:"exclude_fingerprints"`
}

Expand All @@ -91,6 +102,7 @@ func NewReportFlagGroup() *ReportFlagGroup {
Report: &ReportFlag,
Output: &OutputFlag,
Severity: &SeverityFlag,
FailOnSeverity: &FailOnSeverityFlag,
ExcludeFingerprint: &ExcludeFingerprintFlag,
}
}
Expand All @@ -105,6 +117,7 @@ func (f *ReportFlagGroup) Flags() []*Flag {
f.Report,
f.Output,
f.Severity,
f.FailOnSeverity,
f.ExcludeFingerprint,
}
}
Expand Down Expand Up @@ -147,24 +160,13 @@ func (f *ReportFlagGroup) ToOptions() (ReportOptions, error) {
return ReportOptions{}, invalidFormat
}

severity := getStringSlice(f.Severity)
severityMapping := make(map[string]bool)

for _, severityLevel := range severity {
switch severityLevel {
case types.LevelCritical:
severityMapping[types.LevelCritical] = true
case types.LevelHigh:
severityMapping[types.LevelHigh] = true
case types.LevelMedium:
severityMapping[types.LevelMedium] = true
case types.LevelLow:
severityMapping[types.LevelLow] = true
case types.LevelWarning:
severityMapping[types.LevelWarning] = true
default:
return ReportOptions{}, ErrInvalidSeverity
}
severity := getSeverities(f.Severity)
if severity == nil {
return ReportOptions{}, ErrInvalidSeverity
}
failOnSeverity := getSeverities(f.FailOnSeverity)
if failOnSeverity == nil {
return ReportOptions{}, ErrInvalidFailOnSeverity
}

// turn string slice into map for ease of access
Expand All @@ -178,7 +180,8 @@ func (f *ReportFlagGroup) ToOptions() (ReportOptions, error) {
Format: format,
Report: report,
Output: getString(f.Output),
Severity: severityMapping,
Severity: severity,
FailOnSeverity: failOnSeverity,
ExcludeFingerprint: excludeFingerprintsMapping,
}, nil
}
50 changes: 20 additions & 30 deletions internal/report/output/security/security.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,6 @@ var severityColorFns = map[string]func(x ...interface{}) string{
globaltypes.LevelLow: color.New(color.FgBlue).SprintFunc(),
globaltypes.LevelWarning: color.New(color.FgCyan).SprintFunc(),
}
var orderedSeverityLevels = []string{
globaltypes.LevelCritical,
globaltypes.LevelHigh,
globaltypes.LevelMedium,
globaltypes.LevelLow,
globaltypes.LevelWarning,
}

type Findings = map[string][]types.Finding
type IgnoredFindings = map[string][]types.IgnoredFinding
Expand Down Expand Up @@ -89,11 +82,11 @@ func AddReportData(
output.StdErrLog("Evaluating rules")
}

builtInFingerprints, err := evaluateRules(summaryFindings, ignoredSummaryFindings, config.BuiltInRules, config, dataflow, baseBranchFindings, true)
builtInFingerprints, builtInFailed, err := evaluateRules(summaryFindings, ignoredSummaryFindings, config.BuiltInRules, config, dataflow, baseBranchFindings, true)
if err != nil {
return err
}
fingerprints, err := evaluateRules(summaryFindings, ignoredSummaryFindings, config.Rules, config, dataflow, baseBranchFindings, false)
fingerprints, failed, err := evaluateRules(summaryFindings, ignoredSummaryFindings, config.Rules, config, dataflow, baseBranchFindings, false)
if err != nil {
return err
}
Expand All @@ -109,17 +102,9 @@ func AddReportData(
)
}

// fail the report if we have failures above the severity threshold
reportFailed := false
for severityLevel, findings := range summaryFindings {
if severityLevel != globaltypes.LevelWarning && len(findings) != 0 {
reportFailed = true
}
}

reportData.FindingsBySeverity = summaryFindings
reportData.IgnoredFindingsBySeverity = ignoredSummaryFindings
reportData.ReportFailed = reportFailed
reportData.ReportFailed = builtInFailed || failed
return nil
}

Expand All @@ -131,7 +116,7 @@ func evaluateRules(
dataflow *outputtypes.DataFlow,
baseBranchFindings *basebranchfindings.Findings,
builtIn bool,
) ([]string, error) {
) ([]string, bool, error) {
outputFindings := map[string][]types.Finding{}
ignoredOutputFindings := map[string][]types.IgnoredFinding{}

Expand All @@ -141,6 +126,7 @@ func evaluateRules(
}

var fingerprints []string
failed := false

for _, rule := range maputil.ToSortedSlice(rules) {
if !builtIn {
Expand All @@ -166,19 +152,19 @@ func evaluateRules(
// TODO: perf question: can we do this once?
policy.Modules.ToRegoModules())
if err != nil {
return fingerprints, err
return fingerprints, false, err
}

if len(rs) > 0 {
jsonRes, err := json.Marshal(rs)
if err != nil {
return fingerprints, err
return fingerprints, false, err
}

var results map[string][]Output
err = json.Unmarshal(jsonRes, &results)
if err != nil {
return fingerprints, err
return fingerprints, false, err
}

ruleSummary := &types.Rule{
Expand Down Expand Up @@ -236,22 +222,26 @@ func evaluateRules(
severityMeta := CalculateSeverity(finding.CategoryGroups, rule.GetSeverity(), output.IsLocal != nil && *output.IsLocal)
severity := severityMeta.DisplaySeverity

if config.Report.Severity[severity] {
if config.Report.Severity.Has(severity) {
finding.SeverityMeta = severityMeta
if ignored {
ignoredOutputFindings[severity] = append(ignoredOutputFindings[severity], types.IgnoredFinding{Finding: finding, IgnoreMeta: ignoredFingerprint})
} else {
outputFindings[severity] = append(outputFindings[severity], finding)
}
}

if config.Report.FailOnSeverity.Has(severity) && !ignored {
failed = true
}
}
}
}

sortFindingsBySeverity(summaryFindings, outputFindings)
sortFindingsBySeverity(ignoredSummaryFindings, ignoredOutputFindings)

return fingerprints, nil
return fingerprints, failed, nil
}

func sortFindingsBySeverity[F types.GenericFinding](findingsBySeverity map[string][]F, outputFindings map[string][]F) {
Expand Down Expand Up @@ -400,7 +390,7 @@ func BuildReportString(reportData *outputtypes.ReportData, config settings.Confi
globaltypes.LevelWarning: make(map[string]bool),
}

for _, severityLevel := range orderedSeverityLevels {
for _, severityLevel := range globaltypes.Severities {
for _, failure := range reportData.FindingsBySeverity[severityLevel] {
for i := 0; i < len(failure.CWEIDs); i++ {
failures[severityLevel]["CWE-"+failure.CWEIDs[i]] = true
Expand Down Expand Up @@ -661,7 +651,7 @@ func checkAndWriteFailureSummaryToString(
findings Findings,
ruleCount int,
failures map[string]map[string]bool,
severityForFailure map[string]bool,
reportedSeverity set.Set[string],
) bool {
reportStr.WriteString("\n=====================================")

Expand All @@ -672,7 +662,7 @@ func checkAndWriteFailureSummaryToString(
// give summary including counts
failureCount := 0
warningCount := 0
for _, severityLevel := range maps.Keys(severityForFailure) {
for _, severityLevel := range globaltypes.Severities {
if severityLevel == globaltypes.LevelWarning {
warningCount += len(findings[severityLevel])
continue
Expand All @@ -687,8 +677,8 @@ func checkAndWriteFailureSummaryToString(
reportStr.WriteString("\n\n")
reportStr.WriteString(color.RedString(fmt.Sprint(ruleCount) + " checks, " + fmt.Sprint(failureCount+warningCount) + " findings\n"))

for _, severityLevel := range orderedSeverityLevels {
if !severityForFailure[severityLevel] {
for _, severityLevel := range globaltypes.Severities {
if !reportedSeverity.Has(severityLevel) {
continue
}
reportStr.WriteString("\n" + formatSeverity(severityLevel) + fmt.Sprint(len(findings[severityLevel])))
Expand Down Expand Up @@ -756,7 +746,7 @@ func removeDuplicates[F types.GenericFinding](data map[string][]F) map[string][]
reportedDetections := set.Set[key]{}

// filter duplicates
for _, severity := range orderedSeverityLevels {
for _, severity := range globaltypes.Severities {
findingsSlice, hasSeverity := data[severity]
if !hasSeverity {
continue
Expand Down
Loading

0 comments on commit 0557383

Please sign in to comment.