diff --git a/.envrc.example b/.envrc.example index 4ff592814..8f72545b7 100644 --- a/.envrc.example +++ b/.envrc.example @@ -5,7 +5,7 @@ export GOOGLE_APP='' export REDIS_INIT="true" export GOOGLE_MAX_ATTEMPT="5" export BEARER_EXECUTABLE_PATH="./bearer" -export GITHUB_WORKSPACE="/path/to/bearer/project" +export GITHUB_WORKSPACE="$PWD" export SCAN_DIR=/Users/username/OWASP export BEARER_DISABLE_VERSION_CHECK=true export BEARER_DISABLE_DEFAULT_RULES=true diff --git a/cmd/bearer/bearer.go b/cmd/bearer/bearer.go index 76767d431..5286ac2cf 100644 --- a/cmd/bearer/bearer.go +++ b/cmd/bearer/bearer.go @@ -1,16 +1,10 @@ package main import ( - "os" - "github.com/bearer/bearer/cmd/bearer/build" - "github.com/bearer/bearer/internal/commands" + "github.com/bearer/bearer/external/run" ) func main() { - app := commands.NewApp(build.Version, build.CommitSHA) - if err := app.Execute(); err != nil { - // error messages are printed by the framework - os.Exit(1) - } + run.Run(build.Version, build.CommitSHA, run.NewEngine(run.DefaultLanguages())) } diff --git a/external/run/run.go b/external/run/run.go new file mode 100644 index 000000000..a9a4996cb --- /dev/null +++ b/external/run/run.go @@ -0,0 +1,32 @@ +package run + +import ( + "os" + + "github.com/bearer/bearer/internal/commands" + "github.com/bearer/bearer/internal/engine" + engineimpl "github.com/bearer/bearer/internal/engine/implementation" + "github.com/bearer/bearer/internal/languages" + "github.com/bearer/bearer/internal/scanner/language" +) + +type Language = language.Language +type Engine = engine.Engine + +func NewEngine(languages []Language) Engine { + return engineimpl.New(languages) +} + +func DefaultLanguages() []Language { + return languages.Default() +} + +func Run(version, commitSHA string, engine Engine) { + err := commands.NewApp(version, commitSHA, engine).Execute() + engine.Close() + + if err != nil { + // error messages are printed by the framework + os.Exit(1) + } +} diff --git a/internal/commands/app.go b/internal/commands/app.go index d90894643..8aa826ed1 100644 --- a/internal/commands/app.go +++ b/internal/commands/app.go @@ -6,16 +6,17 @@ import ( "github.com/spf13/cobra" "github.com/bearer/bearer/cmd/bearer/build" + "github.com/bearer/bearer/internal/engine" ) // NewApp is the factory method to return CLI -func NewApp(version string, commitSHA string) *cobra.Command { +func NewApp(version string, commitSHA string, engine engine.Engine) *cobra.Command { rootCmd := NewRootCommand() rootCmd.AddCommand( NewCompletionCommand(), NewProcessingWorkerCommand(), NewInitCommand(), - NewScanCommand(), + NewScanCommand(engine), NewIgnoreCommand(), NewVersionCommand(version, commitSHA), ) diff --git a/internal/commands/artifact/run.go b/internal/commands/artifact/run.go index ce5b3aaac..69be9c8bf 100644 --- a/internal/commands/artifact/run.go +++ b/internal/commands/artifact/run.go @@ -20,9 +20,9 @@ import ( "github.com/bearer/bearer/internal/commands/process/filelist" "github.com/bearer/bearer/internal/commands/process/filelist/files" "github.com/bearer/bearer/internal/commands/process/gitrepository" - "github.com/bearer/bearer/internal/commands/process/orchestrator" - "github.com/bearer/bearer/internal/commands/process/orchestrator/work" "github.com/bearer/bearer/internal/commands/process/settings" + settingsloader "github.com/bearer/bearer/internal/commands/process/settings/loader" + "github.com/bearer/bearer/internal/engine" "github.com/bearer/bearer/internal/flag" flagtypes "github.com/bearer/bearer/internal/flag/types" "github.com/bearer/bearer/internal/report/basebranchfindings" @@ -71,6 +71,7 @@ type runner struct { scanSettings settings.Config stats *scannerstats.Stats gitContext *gitrepository.Context + engine engine.Engine } // NewRunner initializes Runner that provides scanning functionalities. @@ -81,6 +82,7 @@ func NewRunner( targetPath string, goclocResult *gocloc.Result, stats *scannerstats.Stats, + engine engine.Engine, ) (Runner, error) { r := &runner{ scanSettings: scanSettings, @@ -88,6 +90,7 @@ func NewRunner( goclocResult: goclocResult, stats: stats, gitContext: gitContext, + engine: engine, } scanID, err := scanid.Build(scanSettings, gitContext) @@ -160,24 +163,13 @@ func (r *runner) Scan(ctx context.Context, opts flagtypes.Options) ([]files.File return nil, nil, err } - orchestrator, err := orchestrator.New( - work.Repository{Dir: r.targetPath}, - r.scanSettings, - r.stats, - len(fileList.Files), - ) - if err != nil { - return nil, nil, err - } - defer orchestrator.Close() - var baseBranchFindings *basebranchfindings.Findings if err := repository.WithBaseBranch(func() error { if !opts.Quiet { outputhandler.StdErrLog(fmt.Sprintf("\nScanning base branch %s", r.gitContext.BaseBranch)) } - baseBranchFindings, err = r.scanBaseBranch(orchestrator, fileList) + baseBranchFindings, err = r.scanBaseBranch(fileList) if err != nil { return err } @@ -191,24 +183,21 @@ func (r *runner) Scan(ctx context.Context, opts flagtypes.Options) ([]files.File return nil, nil, err } - if err := orchestrator.Scan(r.reportPath, fileList.Files); err != nil { + if err := r.engine.Scan(r.stats, r.reportPath, r.targetPath, fileList.Files); err != nil { return nil, nil, err } return fileList.Files, baseBranchFindings, nil } -func (r *runner) scanBaseBranch( - orchestrator *orchestrator.Orchestrator, - fileList *files.List, -) (*basebranchfindings.Findings, error) { +func (r *runner) scanBaseBranch(fileList *files.List) (*basebranchfindings.Findings, error) { result := basebranchfindings.New(fileList) if len(fileList.BaseFiles) == 0 { return result, nil } - if err := orchestrator.Scan(r.reportPath+".base", fileList.BaseFiles); err != nil { + if err := r.engine.Scan(r.stats, r.reportPath+".base", r.targetPath, fileList.BaseFiles); err != nil { return nil, err } @@ -263,7 +252,7 @@ func getIgnoredFingerprints(client *api.API, settings settings.Config, gitContex } // Run performs artifact scanning -func Run(ctx context.Context, opts flagtypes.Options) (err error) { +func Run(ctx context.Context, opts flagtypes.Options, engine engine.Engine) (err error) { targetPath, err := file.CanonicalPath(opts.Target) if err != nil { return fmt.Errorf("failed to get absolute target: %w", err) @@ -302,7 +291,7 @@ func Run(ctx context.Context, opts flagtypes.Options) (err error) { outputhandler.StdErrLog("Loading rules") } - scanSettings, err := settings.FromOptions(opts, versionMeta) + scanSettings, err := settingsloader.FromOptions(opts, versionMeta, engine) scanSettings.Target = opts.Target if err != nil { return err @@ -317,6 +306,10 @@ func Run(ctx context.Context, opts flagtypes.Options) (err error) { return err } + if err := engine.Initialize(&scanSettings); err != nil { + return fmt.Errorf("failed to initialize engine: %w", err) + } + ctx, cancel := context.WithTimeout(ctx, scanSettings.Worker.Timeout) defer cancel() @@ -331,7 +324,7 @@ func Run(ctx context.Context, opts flagtypes.Options) (err error) { stats = scannerstats.New() } - r, err := NewRunner(ctx, scanSettings, gitContext, targetPath, inputgocloc, stats) + r, err := NewRunner(ctx, scanSettings, gitContext, targetPath, inputgocloc, stats, engine) if err != nil { return err } @@ -407,11 +400,7 @@ func (r *runner) Report( endTime := time.Now() - reportSupported, err := anySupportedLanguagesPresent(report.Inputgocloc, r.scanSettings) - if err != nil { - return false, err - } - + reportSupported := anySupportedLanguagesPresent(r.engine, report.Inputgocloc, r.scanSettings) if !reportSupported && r.scanSettings.Report.Report != flag.ReportPrivacy && !r.scanSettings.Scan.Quiet { var placeholderStr *strings.Builder placeholderStr, err = getPlaceholderOutput(reportData, report, r.scanSettings, report.Inputgocloc) @@ -426,6 +415,7 @@ func (r *runner) Report( formatStr, err := reportoutput.FormatOutput( reportData, r.scanSettings, + r.engine, report.Inputgocloc, startTime, endTime, @@ -463,9 +453,9 @@ func (r *runner) ReportPath() string { return r.reportPath } -func anySupportedLanguagesPresent(inputgocloc *gocloc.Result, config settings.Config) (bool, error) { +func anySupportedLanguagesPresent(engine engine.Engine, inputgocloc *gocloc.Result, config settings.Config) bool { if inputgocloc == nil { - return true, nil + return true } ruleLanguages := make(map[string]bool) @@ -480,16 +470,14 @@ func anySupportedLanguagesPresent(inputgocloc *gocloc.Result, config settings.Co foundLanguages[strings.ToLower(language.Name)] = true } - for _, supportedLanguage := range maps.Keys(settings.GetSupportedRuleLanguages()) { - _, supportedLangPresent := foundLanguages[supportedLanguage] - - if supportedLangPresent && settings.GetSupportedRuleLanguages()[supportedLanguage] { - return true, nil + for _, supportedLanguage := range engine.GetLanguages() { + if _, supportedLangPresent := foundLanguages[supportedLanguage.ID()]; supportedLangPresent { + return true } } log.Debug().Msg("No language found for which rules are applicable") - return false, nil + return false } func getPlaceholderOutput(reportData *outputtypes.ReportData, report types.Report, config settings.Config, inputgocloc *gocloc.Result) (outputStr *strings.Builder, err error) { diff --git a/internal/commands/init.go b/internal/commands/init.go index f4c0c2020..3a2b39246 100644 --- a/internal/commands/init.go +++ b/internal/commands/init.go @@ -12,7 +12,7 @@ func NewInitCommand() *cobra.Command { Use: "init", Short: "Generates a default config to `bearer.yml`", RunE: func(cmd *cobra.Command, args []string) error { - if err := ScanFlags.BindForConfigInit(NewScanCommand()); err != nil { + if err := ScanFlags.BindForConfigInit(NewScanCommand(nil)); err != nil { return fmt.Errorf("flag bind error: %w", err) } diff --git a/internal/commands/process/orchestrator/orchestrator.go b/internal/commands/process/orchestrator/orchestrator.go index d65809ad6..5cb66581b 100644 --- a/internal/commands/process/orchestrator/orchestrator.go +++ b/internal/commands/process/orchestrator/orchestrator.go @@ -24,7 +24,7 @@ import ( type Orchestrator struct { repository work.Repository - config settings.Config + config *settings.Config maxWorkersSemaphore chan struct{} done chan struct{} pool *pool.Pool @@ -33,7 +33,7 @@ type Orchestrator struct { func New( repository work.Repository, - config settings.Config, + config *settings.Config, stats *stats.Stats, estimatedFileCount int, ) (*Orchestrator, error) { @@ -202,7 +202,7 @@ func (orchestrator *Orchestrator) writeFileError(reportFile *os.File, file files orchestrator.reportMutex.Unlock() } -func getParallel(fileCount int, config settings.Config) int { +func getParallel(fileCount int, config *settings.Config) int { if config.Scan.Parallel != 0 { return config.Scan.Parallel } diff --git a/internal/commands/process/orchestrator/pool/pool.go b/internal/commands/process/orchestrator/pool/pool.go index d756983b3..e40ad79cf 100644 --- a/internal/commands/process/orchestrator/pool/pool.go +++ b/internal/commands/process/orchestrator/pool/pool.go @@ -25,7 +25,7 @@ type Pool struct { available []*Process } -func New(config settings.Config, stats *stats.Stats) *Pool { +func New(config *settings.Config, stats *stats.Stats) *Pool { executable, err := os.Executable() if err != nil { output.Fatal(fmt.Sprintf("failed to get current command executable %s", err)) diff --git a/internal/commands/process/orchestrator/pool/process.go b/internal/commands/process/orchestrator/pool/process.go index cbd30561f..072ba642d 100644 --- a/internal/commands/process/orchestrator/pool/process.go +++ b/internal/commands/process/orchestrator/pool/process.go @@ -46,7 +46,7 @@ type Process struct { type ProcessOptions struct { executable string baseArguments []string - config settings.Config + config *settings.Config } func newProcess(options *ProcessOptions, id string) (*Process, error) { @@ -88,7 +88,7 @@ func newProcess(options *ProcessOptions, id string) (*Process, error) { return process, nil } -func (process *Process) start(config settings.Config) error { +func (process *Process) start(config *settings.Config) error { if err := process.command.Start(); err != nil { close(process.exitChannel) return err @@ -201,7 +201,7 @@ func (process *Process) reduceMemoryUsage() { } } -func (process *Process) initialize(config settings.Config) error { +func (process *Process) initialize(config *settings.Config) error { log.Debug().Msgf("%s initializing", process.id) start := time.Now() killTime := time.Now().Add(config.Worker.TimeoutWorkerOnline) diff --git a/internal/commands/process/settings/loader/loader.go b/internal/commands/process/settings/loader/loader.go new file mode 100644 index 000000000..9ae630f2f --- /dev/null +++ b/internal/commands/process/settings/loader/loader.go @@ -0,0 +1,78 @@ +package loader + +import ( + "errors" + "fmt" + "slices" + + "github.com/bearer/bearer/internal/commands/process/settings" + "github.com/bearer/bearer/internal/commands/process/settings/policies" + "github.com/bearer/bearer/internal/commands/process/settings/rules" + "github.com/bearer/bearer/internal/engine" + "github.com/bearer/bearer/internal/flag" + flagtypes "github.com/bearer/bearer/internal/flag/types" + "github.com/bearer/bearer/internal/util/ignore" + "github.com/bearer/bearer/internal/version_check" +) + +func FromOptions( + opts flagtypes.Options, + versionMeta *version_check.VersionMeta, + engine engine.Engine, +) (settings.Config, error) { + policies, err := policies.Load() + if err != nil { + return settings.Config{}, fmt.Errorf("failed to load policies: %w", err) + } + + result, err := rules.Load( + opts.ExternalRuleDir, + opts.RuleOptions, + versionMeta, + engine, + opts.ScanOptions.Force, + ) + if err != nil { + return settings.Config{}, err + } + + ignoredFingerprints, _, _, err := ignore.GetIgnoredFingerprints(opts.GeneralOptions.IgnoreFile, &opts.ScanOptions.Target) + if err != nil { + return settings.Config{}, err + } + + config := settings.Config{ + Client: opts.Client, + Worker: settings.WorkerOptions{ + Timeout: settings.Timeout, + TimeoutFileMinimum: settings.TimeoutFileMinimum, + TimeoutFileMaximum: settings.TimeoutFileMaximum, + TimeoutFileBytesPerSecond: settings.TimeoutFileBytesPerSecond, + TimeoutWorkerOnline: settings.TimeoutWorkerOnline, + FileSizeMaximum: settings.FileSizeMaximum, + ExistingWorker: settings.ExistingWorker, + }, + Scan: opts.ScanOptions, + Report: opts.ReportOptions, + IgnoredFingerprints: ignoredFingerprints, + NoColor: opts.GeneralOptions.NoColor || opts.ReportOptions.Output != "", + DebugProfile: opts.GeneralOptions.DebugProfile, + Debug: opts.GeneralOptions.Debug, + LogLevel: opts.GeneralOptions.LogLevel, + IgnoreFile: opts.GeneralOptions.IgnoreFile, + IgnoreGit: opts.GeneralOptions.IgnoreGit, + Policies: policies, + Rules: result.Rules, + BuiltInRules: result.BuiltInRules, + CacheUsed: result.CacheUsed, + BearerRulesVersion: result.BearerRulesVersion, + } + + if config.Scan.Diff { + if !slices.Contains([]string{flag.ReportSecurity, flag.ReportSaaS}, config.Report.Report) { + return settings.Config{}, errors.New("diff base branch is only supported for the security report") + } + } + + return config, nil +} diff --git a/internal/commands/process/settings/policies/policies.go b/internal/commands/process/settings/policies/policies.go new file mode 100644 index 000000000..3f9789871 --- /dev/null +++ b/internal/commands/process/settings/policies/policies.go @@ -0,0 +1,54 @@ +package policies + +import ( + "embed" + "fmt" + + "gopkg.in/yaml.v2" + + "github.com/bearer/bearer/internal/commands/process/settings" +) + +//go:embed *.rego +var policiesFS embed.FS + +func Load() (map[string]*settings.Policy, error) { + policies, err := loadDefaultPolicies() + if err != nil { + return nil, err + } + + for _, policy := range policies { + for _, module := range policy.Modules { + if module.Path != "" { + content, err := policiesFS.ReadFile(module.Path) + if err != nil { + return nil, err + } + + module.Content = string(content) + } + } + } + + return policies, nil +} + +//go:embed policies.yml +var defaultPolicies []byte + +func loadDefaultPolicies() (map[string]*settings.Policy, error) { + policiesByType := make(map[string]*settings.Policy) + + var policies []*settings.Policy + err := yaml.Unmarshal(defaultPolicies, &policies) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal policy file %s", err) + } + + for _, policy := range policies { + policiesByType[policy.Type] = policy + } + + return policiesByType, nil +} diff --git a/internal/commands/process/settings/policies.yml b/internal/commands/process/settings/policies/policies.yml similarity index 71% rename from internal/commands/process/settings/policies.yml rename to internal/commands/process/settings/policies/policies.yml index 4d8f10565..aefaece06 100644 --- a/internal/commands/process/settings/policies.yml +++ b/internal/commands/process/settings/policies/policies.yml @@ -3,15 +3,15 @@ policy_failure = data.bearer.risk_policy.policy_failure local_rule_failure = data.bearer.risk_policy.local_rule_failure modules: - - path: policies/common.rego + - path: common.rego name: bearer.common - - path: policies/risk_policy.rego + - path: risk_policy.rego name: bearer.risk_policy - type: privacy_report query: | items = data.bearer.privacy_report.items modules: - - path: policies/common.rego + - path: common.rego name: bearer.common - - path: policies/privacy_report.rego + - path: privacy_report.rego name: bearer.privacy_report diff --git a/internal/commands/process/settings/processors/processors.go b/internal/commands/process/settings/processors/processors.go new file mode 100644 index 000000000..70325e601 --- /dev/null +++ b/internal/commands/process/settings/processors/processors.go @@ -0,0 +1,23 @@ +package processors + +import ( + "embed" + "fmt" + + "github.com/bearer/bearer/internal/util/rego" +) + +//go:embed *.rego +var fs embed.FS + +func Load(name string) ([]rego.Module, error) { + moduleText, err := fs.ReadFile(fmt.Sprintf("%s.rego", name)) + if err != nil { + return nil, err + } + + return []rego.Module{{ + Name: fmt.Sprintf("bearer.%s", name), + Content: string(moduleText), + }}, nil +} diff --git a/internal/commands/process/settings/ruleLoader.go b/internal/commands/process/settings/ruleLoader.go deleted file mode 100644 index 48deb68c9..000000000 --- a/internal/commands/process/settings/ruleLoader.go +++ /dev/null @@ -1,142 +0,0 @@ -package settings - -import ( - "archive/tar" - "compress/gzip" - "crypto/md5" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/rs/zerolog/log" - "gopkg.in/yaml.v3" -) - -const BASE_RULE_FOLDER = "/" - -func LoadRuleDefinitionsFromUrls(ruleDefinitions map[string]RuleDefinition, languageDownloads []string) (err error) { - - bearerRulesDir := bearerRulesDir() - if _, err := os.Stat(bearerRulesDir); errors.Is(err, os.ErrNotExist) { - err := os.Mkdir(bearerRulesDir, os.ModePerm) - if err != nil { - return fmt.Errorf("could not create bearer-rules directory: %s", err) - } - } - - for _, languagePackageUrl := range languageDownloads { - // Prepare filepath - urlHash := md5.Sum([]byte(languagePackageUrl)) - filepath, err := filepath.Abs(filepath.Join(bearerRulesDir, fmt.Sprintf("%x.tar.gz", urlHash))) - - if err != nil { - return err - } - - if _, err := os.Stat(filepath); err == nil { - log.Trace().Msgf("Using local cache for rule package: %s", languagePackageUrl) - file, err := os.Open(filepath) - if err != nil { - return err - } - defer file.Close() - - if err = ReadRuleDefinitions(ruleDefinitions, file); err != nil { - return err - } - } else { - log.Trace().Msgf("Downloading rule package: %s", languagePackageUrl) - httpClient := &http.Client{Timeout: 60 * time.Second} - resp, err := httpClient.Get(languagePackageUrl) - if err != nil { - return err - } - defer resp.Body.Close() - - // Create file in rules dir - file, err := os.Create(filepath) - if err != nil { - return err - } - defer file.Close() - - // Copy the contents of the downloaded archive to the file - if _, err := io.Copy(file, resp.Body); err != nil { - return err - } - // reset file pointer to start of file - _, err = file.Seek(0, 0) - if err != nil { - return err - } - - if err = ReadRuleDefinitions(ruleDefinitions, file); err != nil { - return err - } - } - } - - return nil -} - -func ReadRuleDefinitions(ruleDefinitions map[string]RuleDefinition, file *os.File) error { - gzr, err := gzip.NewReader(file) - if err != nil { - return err - } - defer gzr.Close() - - tr := tar.NewReader(gzr) - for { - header, err := tr.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - - if !isRuleFile(header.Name) { - continue - } - - data := make([]byte, header.Size) - _, err = io.ReadFull(tr, data) - if err != nil { - return fmt.Errorf("failed to read file %s: %w", header.Name, err) - } - - var ruleDefinition RuleDefinition - err = yaml.Unmarshal(data, &ruleDefinition) - if err != nil { - return fmt.Errorf("failed to unmarshal rule %s: %w", header.Name, err) - } - - id := ruleDefinition.Metadata.ID - _, ruleExists := ruleDefinitions[id] - if ruleExists { - return fmt.Errorf("duplicate built-in rule ID %s", id) - } - - ruleDefinitions[id] = ruleDefinition - } - - return nil -} - -func isRuleFile(headerName string) bool { - if strings.Contains(headerName, ".snapshots") { - return false - } - - ext := filepath.Ext(headerName) - if ext != ".yaml" && ext != ".yml" { - return false - } - - return strings.Contains(headerName, BASE_RULE_FOLDER) -} diff --git a/internal/commands/process/settings/ruleValidator.go b/internal/commands/process/settings/ruleValidator.go deleted file mode 100644 index a0077425c..000000000 --- a/internal/commands/process/settings/ruleValidator.go +++ /dev/null @@ -1,68 +0,0 @@ -package settings - -import ( - "bytes" - "fmt" - "net/http" - "strings" - - "github.com/xeipuuv/gojsonschema" - "sigs.k8s.io/yaml" -) - -const SCHEMA_URL = "https://raw.githubusercontent.com/Bearer/bearer-rules/main/scripts/rule_schema.json" - -func ValidateRule(entry []byte, filename string) string { - validationStr := &strings.Builder{} - validationStr.WriteString(fmt.Sprintf("Failed to load %s\nValidating against %s\n\n", filename, SCHEMA_URL)) - schema, err := loadSchema(SCHEMA_URL) - if err != nil { - validationStr.WriteString("Could not load schema to validate") - return validationStr.String() - } - - jsonData, err := yaml.YAMLToJSON(entry) - if err != nil { - validationStr.WriteString("File format is invalid") - return validationStr.String() - } - - result, err := validateData(jsonData, schema) - if err != nil { - validationStr.WriteString("Could not apply validation") - return validationStr.String() - } - - if result.Valid() { - validationStr.WriteString("Format of appears valid but could not be loaded") - } else { - validationStr.WriteString(fmt.Sprintf("%s validation issues found:\n", filename)) - for _, desc := range result.Errors() { - validationStr.WriteString(fmt.Sprintf("- %s\n", desc)) - } - fmt.Print("\n") - } - return validationStr.String() -} - -func loadSchema(url string) (*gojsonschema.Schema, error) { - response, err := http.Get(url) - if err != nil { - return nil, err - } - defer response.Body.Close() - - var buf bytes.Buffer - _, err = buf.ReadFrom(response.Body) - if err != nil { - return nil, err - } - - loader := gojsonschema.NewStringLoader(buf.String()) - return gojsonschema.NewSchema(loader) -} - -func validateData(data []byte, schema *gojsonschema.Schema) (*gojsonschema.Result, error) { - loader := gojsonschema.NewStringLoader(string(data)) - return schema.Validate(loader) -} diff --git a/internal/commands/process/settings/rules.go b/internal/commands/process/settings/rules.go deleted file mode 100644 index fdbd27974..000000000 --- a/internal/commands/process/settings/rules.go +++ /dev/null @@ -1,485 +0,0 @@ -package settings - -import ( - "fmt" - "io/fs" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/rs/zerolog/log" - "gopkg.in/yaml.v3" - - flagtypes "github.com/bearer/bearer/internal/flag/types" - "github.com/bearer/bearer/internal/report/customdetectors" - "github.com/bearer/bearer/internal/util/output" - "github.com/bearer/bearer/internal/util/set" - "github.com/bearer/bearer/internal/version_check" -) - -const ( - defaultRuleType = customdetectors.TypeRisk - defaultAuxiliaryRuleType = customdetectors.TypeVerifier -) - -var ( - builtinRuleIDs = []string{ - "datatype", - "insecure_url", - "string_literal", - } -) - -func GetSupportedRuleLanguages() map[string]bool { - return map[string]bool{ - "python": true, - "php": true, - "go": true, - "java": true, - "sql": true, // partly supported but not exposed - "ruby": true, - "javascript": true, - "typescript": true, - } -} - -func loadRules( - externalRuleDirs []string, - options flagtypes.RuleOptions, - versionMeta *version_check.VersionMeta, - force bool, -) ( - result LoadRulesResult, - err error, -) { - definitions := make(map[string]RuleDefinition) - builtInDefinitions := make(map[string]RuleDefinition) - - if versionMeta.Rules.Version != nil { - result.BearerRulesVersion = *versionMeta.Rules.Version - } - - log.Debug().Msg("Loading rules") - - loadRuleDefinitionsFromRemote(definitions, options, versionMeta) - - if err := loadRuleDefinitionsFromDir(builtInDefinitions, buildInRulesFs); err != nil { - return result, fmt.Errorf("error loading built-in rules: %w", err) - } - - for _, dir := range externalRuleDirs { - if strings.HasPrefix(dir, "~/") { - dirname, _ := os.UserHomeDir() - dir = filepath.Join(dirname, dir[2:]) - } - log.Debug().Msgf("loading external rules from: %s", dir) - if err := loadRuleDefinitionsFromDir(definitions, os.DirFS(dir)); err != nil { - return result, fmt.Errorf("external rules %w", err) - } - } - - if err := validateRuleOptionIDs(options, definitions, builtInDefinitions); err != nil { - return result, err - } - - enabledRules := getEnabledRules(options, definitions, nil) - builtInRules := getEnabledRules(options, builtInDefinitions, enabledRules) - - result.Rules = BuildRules(definitions, enabledRules) - result.BuiltInRules = BuildRules(builtInDefinitions, builtInRules) - - return result, nil -} - -func loadRuleDefinitionsFromRemote( - definitions map[string]RuleDefinition, - options flagtypes.RuleOptions, - versionMeta *version_check.VersionMeta, -) (err error) { - if options.DisableDefaultRules { - return - } - - if versionMeta.Rules.Version == nil { - log.Debug().Msg("No rule packages found") - return - } - - urls := make([]string, 0, len(versionMeta.Rules.Packages)) - for _, value := range versionMeta.Rules.Packages { - log.Debug().Msgf("Added rule package URL %s", value) - urls = append(urls, value) - } - - err = LoadRuleDefinitionsFromUrls(definitions, urls) - if err != nil { - err = fmt.Errorf("loading rules failed: %s", err) - } - - return -} - -func loadRuleDefinitionsFromDir(definitions map[string]RuleDefinition, dir fs.FS) error { - loadedDefinitions := make(map[string]RuleDefinition) - if err := fs.WalkDir(dir, ".", func(path string, dirEntry fs.DirEntry, err error) error { - if err != nil { - return err - } - - if dirEntry.IsDir() { - return nil - } - - filename := dirEntry.Name() - ext := filepath.Ext(filename) - - if ext != ".yaml" && ext != ".yml" { - return nil - } - - entry, err := fs.ReadFile(dir, path) - if err != nil { - return fmt.Errorf("failed to read file %s: %w", path, err) - } - - var ruleDefinition RuleDefinition - err = yaml.Unmarshal(entry, &ruleDefinition) - if err != nil { - output.StdErrLog(ValidateRule(entry, filename)) - return fmt.Errorf("rule file was invalid") - } - - if ruleDefinition.Metadata == nil { - log.Debug().Msgf("rule file has invalid metadata %s", path) - return nil - } - - id := ruleDefinition.Metadata.ID - if id == "" { - log.Debug().Msgf("rule file missing metadata.id %s", path) - return nil - } - - for _, language := range ruleDefinition.Languages { - if exists := GetSupportedRuleLanguages()[language]; !exists { - log.Debug().Msgf("rule file includes unsupported language[%s] %s", language, path) - return nil - } - } - - if _, exists := loadedDefinitions[id]; exists { - return fmt.Errorf("duplicate rule ID %s", id) - } - - loadedDefinitions[id] = ruleDefinition - - return nil - }); err != nil { - return err - } - - for id, definition := range loadedDefinitions { - if validateRuleDefinition(loadedDefinitions, &definition) { - definitions[id] = definition - } - } - - return nil -} - -func validateRuleDefinition(allDefinitions map[string]RuleDefinition, definition *RuleDefinition) bool { - metadata := definition.Metadata - - valid := true - fail := func(message string) { - valid = false - log.Debug().Msgf("%s: %s", metadata.ID, message) - } - - visibleRuleIDs := set.New[string]() - visibleRuleIDs.Add(metadata.ID) - visibleRuleIDs.AddAll(builtinRuleIDs) - - for _, importedID := range definition.Imports { - visibleRuleIDs.Add(importedID) - - importedDefinition, exists := allDefinitions[importedID] - - if !exists { - fail(fmt.Sprintf("import of unknown rule '%s'", importedID)) - continue - } - - if importedDefinition.Type != customdetectors.TypeShared { - fail(fmt.Sprintf("imported rule '%s' is not of type 'shared'", importedID)) - } - } - - for _, auxiliaryDefinition := range definition.Auxiliary { - visibleRuleIDs.Add(auxiliaryDefinition.Id) - } - - for _, filterRuleID := range getFilterRuleReferences(definition).Items() { - if !visibleRuleIDs.Has(filterRuleID) { - fail(fmt.Sprintf("filter references invalid or non-imported rule '%s'", filterRuleID)) - } - } - - for _, sanitizerRuleID := range getSanitizers(definition).Items() { - if !visibleRuleIDs.Has(sanitizerRuleID) { - fail(fmt.Sprintf("sanitizer references invalid or non-imported rule '%s'", sanitizerRuleID)) - } - } - - if metadata.ID == "" { - fail("metadata.id must be specified") - } - - if definition.Type == customdetectors.TypeShared { - metadata := definition.Metadata - if metadata != nil { - if metadata.CWEIDs != nil { - fail("cwe ids cannot be specified for a shared rule") - } - - if metadata.RemediationMessage != "" { - fail("remediation message cannot be specified for a shared rule") - } - } - - if definition.Severity != "" { - fail("severity cannot be specified for a shared rule") - } - } - - if !valid { - log.Debug().Msgf("%s ignored due to validation errors", metadata.ID) - } - - return valid -} - -func getFilterRuleReferences(definition *RuleDefinition) set.Set[string] { - result := set.New[string]() - - var addFilter func(filter PatternFilter) - - addPatterns := func(patterns []RulePattern) { - for _, pattern := range patterns { - for _, filter := range pattern.Filters { - addFilter(filter) - } - } - } - - addFilter = func(filter PatternFilter) { - if filter.Detection != "" { - result.Add(filter.Detection) - } - - if filter.Not != nil { - addFilter(*filter.Not) - } - - for _, subFilter := range filter.Either { - addFilter(subFilter) - } - } - - addPatterns(definition.Patterns) - for _, auxiliaryDefinition := range definition.Auxiliary { - addPatterns(auxiliaryDefinition.Patterns) - } - - return result -} - -func getSanitizers(definition *RuleDefinition) set.Set[string] { - result := set.New[string]() - - if definition.SanitizerRuleID != "" { - result.Add(definition.SanitizerRuleID) - } - - for _, auxiliaryDefinition := range definition.Auxiliary { - if auxiliaryDefinition.SanitizerRuleID != "" { - result.Add(auxiliaryDefinition.SanitizerRuleID) - } - } - - return result -} - -func validateRuleOptionIDs( - options flagtypes.RuleOptions, - definitions map[string]RuleDefinition, - builtInDefinitions map[string]RuleDefinition, -) error { - var invalidRuleIDs []string - - for id := range options.OnlyRule { - _, existsInDefinition := definitions[id] - _, existsInBuiltInDefinition := builtInDefinitions[id] - - if !existsInBuiltInDefinition && !existsInDefinition { - invalidRuleIDs = append(invalidRuleIDs, id) - } - } - var invalidSkipRuleIDs []string - for id := range options.SkipRule { - _, existsInDefinition := definitions[id] - _, existsInBuiltInDefinition := builtInDefinitions[id] - - if !existsInBuiltInDefinition && !existsInDefinition { - invalidSkipRuleIDs = append(invalidSkipRuleIDs, id) - } - } - - if len(invalidSkipRuleIDs) > 0 { - sort.Strings(invalidSkipRuleIDs) - output.StdErrLog(fmt.Sprintf("Warning: rule IDs %s given to be skipped but were not found", strings.Join(invalidSkipRuleIDs, ","))) - } - if len(invalidRuleIDs) > 0 { - return fmt.Errorf("invalid rule IDs in only option: %s", strings.Join(invalidRuleIDs, ",")) - } - - return nil -} - -func getEnabledRules(options flagtypes.RuleOptions, definitions map[string]RuleDefinition, rules map[string]struct{}) map[string]struct{} { - enabledRules := make(map[string]struct{}) - - for ruleId := range rules { - enabledRules[ruleId] = struct{}{} - } - - var enableRule func(definition RuleDefinition) - enableRule = func(definition RuleDefinition) { - if definition.Disabled { - return - } - - id := definition.Metadata.ID - - if _, alreadyEnabled := enabledRules[id]; alreadyEnabled { - return - } - - enabledRules[id] = struct{}{} - - for _, dependencyID := range definition.Detectors { - enabledRules[dependencyID] = struct{}{} - } - - for _, importedRuleID := range definition.Imports { - if importedDefinition, exists := definitions[importedRuleID]; exists { - enableRule(importedDefinition) - } - } - } - - for _, definition := range definitions { - id := definition.Metadata.ID - - if len(options.OnlyRule) > 0 && !options.OnlyRule[id] { - continue - } - - if options.SkipRule[id] { - continue - } - - enableRule(definition) - } - - return enabledRules -} - -func BuildRules(definitions map[string]RuleDefinition, enabledRules map[string]struct{}) map[string]*Rule { - rules := make(map[string]*Rule) - - for _, definition := range definitions { - id := definition.Metadata.ID - - if _, enabled := enabledRules[id]; !enabled { - continue - } - - ruleType := definition.Type - if len(ruleType) == 0 { - ruleType = defaultRuleType - } - - // build rule trigger - ruleTrigger := RuleTrigger{ - MatchOn: PRESENCE, - DataTypesRequired: false, - } - - if definition.Trigger != nil { - if definition.Trigger.MatchOn != nil { - ruleTrigger.MatchOn = *definition.Trigger.MatchOn - } - if definition.Trigger.DataTypesRequired != nil { - ruleTrigger.DataTypesRequired = *definition.Trigger.DataTypesRequired - } - if definition.Trigger.RequiredDetection != nil { - ruleTrigger.RequiredDetection = definition.Trigger.RequiredDetection - } - } - - isLocal := false - for _, rulePattern := range definition.Patterns { - if strings.Contains(rulePattern.Pattern, "$") { - isLocal = true - break - } - } - - rules[id] = &Rule{ - Id: id, - Type: ruleType, - AssociatedRecipe: definition.Metadata.AssociatedRecipe, - Trigger: ruleTrigger, - IsLocal: isLocal, - SkipDataTypes: definition.SkipDataTypes, - OnlyDataTypes: definition.OnlyDataTypes, - Severity: definition.Severity, - Description: definition.Metadata.Description, - RemediationMessage: definition.Metadata.RemediationMessage, - Stored: definition.Stored, - Detectors: definition.Detectors, - Processors: definition.Processors, - AutoEncrytPrefix: definition.AutoEncrytPrefix, - CWEIDs: definition.Metadata.CWEIDs, - Languages: definition.Languages, - ParamParenting: definition.ParamParenting, - Patterns: definition.Patterns, - SanitizerRuleID: definition.SanitizerRuleID, - DocumentationUrl: definition.Metadata.DocumentationUrl, - HasDetailedContext: definition.HasDetailedContext, - DependencyCheck: definition.DependencyCheck, - Dependency: definition.Dependency, - } - - for _, auxiliaryDefinition := range definition.Auxiliary { - rules[auxiliaryDefinition.Id] = &Rule{ - Id: auxiliaryDefinition.Id, - Type: defaultAuxiliaryRuleType, - Languages: definition.Languages, - ParamParenting: auxiliaryDefinition.ParamParenting, - Patterns: auxiliaryDefinition.Patterns, - SanitizerRuleID: auxiliaryDefinition.SanitizerRuleID, - Stored: auxiliaryDefinition.Stored, - IsAuxilary: true, - } - } - } - - return rules -} - -func bearerRulesDir() string { - return filepath.Join(os.TempDir(), "bearer-rules") -} diff --git a/internal/commands/process/settings/built_in_rules/sql/lang/create_table.yml b/internal/commands/process/settings/rules/built_in/sql/lang/create_table.yml similarity index 100% rename from internal/commands/process/settings/built_in_rules/sql/lang/create_table.yml rename to internal/commands/process/settings/rules/built_in/sql/lang/create_table.yml diff --git a/internal/commands/process/settings/built_in_rules/third_party/gitleaks/secret_detection.yml b/internal/commands/process/settings/rules/built_in/third_party/gitleaks/secret_detection.yml similarity index 100% rename from internal/commands/process/settings/built_in_rules/third_party/gitleaks/secret_detection.yml rename to internal/commands/process/settings/rules/built_in/third_party/gitleaks/secret_detection.yml diff --git a/internal/commands/process/settings/rules/javascript/lang/jwt/.snapshots/TestJavascriptJWT--jwt_insecure.yml b/internal/commands/process/settings/rules/javascript/lang/jwt/.snapshots/TestJavascriptJWT--jwt_insecure.yml deleted file mode 100644 index 311847daa..000000000 --- a/internal/commands/process/settings/rules/javascript/lang/jwt/.snapshots/TestJavascriptJWT--jwt_insecure.yml +++ /dev/null @@ -1,2 +0,0 @@ -{} - diff --git a/internal/commands/process/settings/rules/javascript/lang/jwt/testdata/jwt_insecure.js b/internal/commands/process/settings/rules/javascript/lang/jwt/testdata/jwt_insecure.js deleted file mode 100644 index 95716c53d..000000000 --- a/internal/commands/process/settings/rules/javascript/lang/jwt/testdata/jwt_insecure.js +++ /dev/null @@ -1,11 +0,0 @@ -import myJWT from "jsonwebtoken"; - -import {jwt as myJWT} from "jsonwebtoken"; - -const myJWT = require("jsonwebtoken").jwt; - -const privateKey = "foo"; -myJWT.sign(user, privateKey, { - expiresInMinutes: 60 * 5, - algorithm: "RS256", -}); diff --git a/internal/commands/process/settings/rules/loader.go b/internal/commands/process/settings/rules/loader.go new file mode 100644 index 000000000..73b266c72 --- /dev/null +++ b/internal/commands/process/settings/rules/loader.go @@ -0,0 +1,242 @@ +package rules + +import ( + "archive/tar" + "compress/gzip" + "crypto/md5" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/rs/zerolog/log" + "gopkg.in/yaml.v3" + + "github.com/bearer/bearer/internal/commands/process/settings" + "github.com/bearer/bearer/internal/engine" + flagtypes "github.com/bearer/bearer/internal/flag/types" + "github.com/bearer/bearer/internal/util/output" + "github.com/bearer/bearer/internal/version_check" +) + +func loadDefinitionsFromRemote( + definitions map[string]settings.RuleDefinition, + options flagtypes.RuleOptions, + versionMeta *version_check.VersionMeta, +) error { + if options.DisableDefaultRules { + return nil + } + + if versionMeta.Rules.Version == nil { + log.Debug().Msg("No rule packages found") + return nil + } + + urls := make([]string, 0, len(versionMeta.Rules.Packages)) + for _, value := range versionMeta.Rules.Packages { + log.Debug().Msgf("Added rule package URL %s", value) + urls = append(urls, value) + } + + if err := readDefinitionsFromUrls(definitions, urls); err != nil { + return fmt.Errorf("loading rules failed: %s", err) + } + + return nil +} + +func readDefinitionsFromUrls(ruleDefinitions map[string]settings.RuleDefinition, languageDownloads []string) (err error) { + bearerRulesDir := bearerRulesDir() + if _, err := os.Stat(bearerRulesDir); errors.Is(err, os.ErrNotExist) { + err := os.Mkdir(bearerRulesDir, os.ModePerm) + if err != nil { + return fmt.Errorf("could not create bearer-rules directory: %s", err) + } + } + + for _, languagePackageUrl := range languageDownloads { + // Prepare filepath + urlHash := md5.Sum([]byte(languagePackageUrl)) + filepath, err := filepath.Abs(filepath.Join(bearerRulesDir, fmt.Sprintf("%x.tar.gz", urlHash))) + + if err != nil { + return err + } + + if _, err := os.Stat(filepath); err == nil { + log.Trace().Msgf("Using local cache for rule package: %s", languagePackageUrl) + file, err := os.Open(filepath) + if err != nil { + return err + } + defer file.Close() + + if err = readRuleDefinitionZip(ruleDefinitions, file); err != nil { + return err + } + } else { + log.Trace().Msgf("Downloading rule package: %s", languagePackageUrl) + httpClient := &http.Client{Timeout: 60 * time.Second} + resp, err := httpClient.Get(languagePackageUrl) + if err != nil { + return err + } + defer resp.Body.Close() + + // Create file in rules dir + file, err := os.Create(filepath) + if err != nil { + return err + } + defer file.Close() + + // Copy the contents of the downloaded archive to the file + if _, err := io.Copy(file, resp.Body); err != nil { + return err + } + // reset file pointer to start of file + _, err = file.Seek(0, 0) + if err != nil { + return err + } + + if err = readRuleDefinitionZip(ruleDefinitions, file); err != nil { + return err + } + } + } + + return nil +} + +func readRuleDefinitionZip(ruleDefinitions map[string]settings.RuleDefinition, file *os.File) error { + gzr, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gzr.Close() + + tr := tar.NewReader(gzr) + for { + header, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + if !isRuleFile(header.Name) { + continue + } + + data := make([]byte, header.Size) + _, err = io.ReadFull(tr, data) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", header.Name, err) + } + + var ruleDefinition settings.RuleDefinition + err = yaml.Unmarshal(data, &ruleDefinition) + if err != nil { + return fmt.Errorf("failed to unmarshal rule %s: %w", header.Name, err) + } + + id := ruleDefinition.Metadata.ID + _, ruleExists := ruleDefinitions[id] + if ruleExists { + return fmt.Errorf("duplicate built-in rule ID %s", id) + } + + ruleDefinitions[id] = ruleDefinition + } + + return nil +} + +func loadCustomDefinitions(engine engine.Engine, definitions map[string]settings.RuleDefinition, isBuiltIn bool, dir fs.FS) error { + loadedDefinitions := make(map[string]settings.RuleDefinition) + if err := fs.WalkDir(dir, ".", func(path string, dirEntry fs.DirEntry, err error) error { + if err != nil { + return err + } + + if dirEntry.IsDir() { + return nil + } + + filename := dirEntry.Name() + ext := filepath.Ext(filename) + + if ext != ".yaml" && ext != ".yml" { + return nil + } + + entry, err := fs.ReadFile(dir, path) + if err != nil { + return fmt.Errorf("failed to read file %s: %w", path, err) + } + + var ruleDefinition settings.RuleDefinition + err = yaml.Unmarshal(entry, &ruleDefinition) + if err != nil { + output.StdErrLog(validateCustomRuleSchema(entry, filename)) + return fmt.Errorf("rule file was invalid") + } + + if ruleDefinition.Metadata == nil { + log.Debug().Msgf("rule file has invalid metadata %s", path) + return nil + } + + id := ruleDefinition.Metadata.ID + if id == "" { + log.Debug().Msgf("rule file missing metadata.id %s", path) + return nil + } + + supported := isBuiltIn + for _, languageID := range ruleDefinition.Languages { + language := engine.GetLanguageById(languageID) + if language != nil { + supported = true + } + } + + if !supported { + log.Debug().Msgf( + "rule file has no supported languages[%s] %s", + strings.Join(ruleDefinition.Languages, ", "), + path, + ) + return nil + } + + if _, exists := loadedDefinitions[id]; exists { + return fmt.Errorf("duplicate rule ID %s", id) + } + + loadedDefinitions[id] = ruleDefinition + + return nil + }); err != nil { + return err + } + + for id, definition := range loadedDefinitions { + if validateCustomRuleDefinition(loadedDefinitions, &definition) { + definitions[id] = definition + } + } + + return nil +} + +func bearerRulesDir() string { + return filepath.Join(os.TempDir(), "bearer-rules") +} diff --git a/internal/commands/process/settings/rules/ruby/lang/weak_encryption/.snapshots/TestRubyLangWeakEncryption--sha1.yml b/internal/commands/process/settings/rules/ruby/lang/weak_encryption/.snapshots/TestRubyLangWeakEncryption--sha1.yml deleted file mode 100644 index 0ecc2002a..000000000 --- a/internal/commands/process/settings/rules/ruby/lang/weak_encryption/.snapshots/TestRubyLangWeakEncryption--sha1.yml +++ /dev/null @@ -1,44 +0,0 @@ -high: - - rule: - cwe_ids: - - "331" - - "326" - id: ruby_lang_weak_encryption - title: Weak encryption library usage detected. - description: | - ## Description - - A weak encryption or hashing library can lead to data breaches and greater security risk. This rule checks for the use of weak encryption and hashing libraries or algorithms. - - ## Remediations - According to [OWASP](https://owasp.org/www-project-web-security-testing-guide/latest/4-Web_Application_Security_Testing/09-Testing_for_Weak_Cryptography/04-Testing_for_Weak_Encryption): MD5, RC4, DES, Blowfish, SHA1. 1024-bit RSA or DSA, 160-bit ECDSA (elliptic curves), 80/112-bit 2TDEA (two key triple DES) are considered as weak hash/encryption algorithms and therefor shouldn't be used. - - ❌ Avoid libraries and algorithms with known weaknesses: - - ```ruby - Digest::SHA1.hexdigest 'weak password encryption' - Crypt::Blowfish.new("weak password encryption") - RC4.new("weak password encryption") - OpenSSL::PKey::RSA.new 1024 - OpenSSL::PKey::DSA.new 1024 - Digest::MD5.hexdigest 'unsecure string' - ``` - - ✅ Instead, we recommend using bcrypt: - - ```ruby - BCrypt::Password.create('iLOVEdogs123') - ``` - - ## Resources - - [BCrypt Explained](https://dev.to/sylviapap/bcrypt-explained-4k5c) - documentation_url: https://docs.bearer.com/reference/rules/ruby_lang_weak_encryption - line_number: 1 - filename: sha1.rb - category_groups: - - PII - - Personal Data - parent_line_number: 1 - snippet: Digest::SHA1.hexidigest(user.name) - fingerprint: c7e6bc4967b8598e0d32e8f35f0d5691_0 - diff --git a/internal/commands/process/settings/rules/ruby/lang/weak_encryption/testdata/sha1.rb b/internal/commands/process/settings/rules/ruby/lang/weak_encryption/testdata/sha1.rb deleted file mode 100644 index a01c74013..000000000 --- a/internal/commands/process/settings/rules/ruby/lang/weak_encryption/testdata/sha1.rb +++ /dev/null @@ -1 +0,0 @@ -Digest::SHA1.hexdigest(aaaa.bbbb) \ No newline at end of file diff --git a/internal/commands/process/settings/rules/rules.go b/internal/commands/process/settings/rules/rules.go new file mode 100644 index 000000000..db49ec546 --- /dev/null +++ b/internal/commands/process/settings/rules/rules.go @@ -0,0 +1,290 @@ +package rules + +import ( + "embed" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/rs/zerolog/log" + "gopkg.in/yaml.v2" + + "github.com/bearer/bearer/internal/commands/process/settings" + "github.com/bearer/bearer/internal/engine" + flagtypes "github.com/bearer/bearer/internal/flag/types" + "github.com/bearer/bearer/internal/report/customdetectors" + "github.com/bearer/bearer/internal/util/set" + "github.com/bearer/bearer/internal/version_check" +) + +const ( + defaultRuleType = customdetectors.TypeRisk + defaultAuxiliaryRuleType = customdetectors.TypeVerifier +) + +type LoadRulesResult struct { + BuiltInRules map[string]*settings.Rule + Rules map[string]*settings.Rule + CacheUsed bool + BearerRulesVersion string +} + +//go:embed built_in/* +var builtInRulesFS embed.FS + +func Load( + externalRuleDirs []string, + options flagtypes.RuleOptions, + versionMeta *version_check.VersionMeta, + engine engine.Engine, + force bool, +) ( + result LoadRulesResult, + err error, +) { + definitions := make(map[string]settings.RuleDefinition) + builtInDefinitions := make(map[string]settings.RuleDefinition) + + if versionMeta.Rules.Version != nil { + result.BearerRulesVersion = *versionMeta.Rules.Version + } + + log.Debug().Msg("Loading rules") + + if err := loadDefinitionsFromRemote(definitions, options, versionMeta); err != nil { + return result, fmt.Errorf("error loading remote rules: %w", err) + } + + if err := loadCustomDefinitions(engine, builtInDefinitions, true, builtInRulesFS); err != nil { + return result, fmt.Errorf("error loading built-in rules: %w", err) + } + + for _, dir := range externalRuleDirs { + if strings.HasPrefix(dir, "~/") { + dirname, _ := os.UserHomeDir() + dir = filepath.Join(dirname, dir[2:]) + } + log.Debug().Msgf("loading external rules from: %s", dir) + if err := loadCustomDefinitions(engine, definitions, false, os.DirFS(dir)); err != nil { + return result, fmt.Errorf("external rules %w", err) + } + } + + if err := validateRuleOptionIDs(options, definitions, builtInDefinitions); err != nil { + return result, err + } + + enabledRules := getEnabledRules(options, definitions, nil) + builtInRules := getEnabledRules(options, builtInDefinitions, enabledRules) + + result.Rules = BuildRules(definitions, enabledRules) + result.BuiltInRules = BuildRules(builtInDefinitions, builtInRules) + + for _, definition := range definitions { + id := definition.Metadata.ID + if _, enabled := enabledRules[id]; enabled { + definitionYAML, err := yaml.Marshal(&definition) + if err != nil { + return result, err + } + + if err := engine.LoadRule(string(definitionYAML)); err != nil { + return result, fmt.Errorf("engine failed to load rule %s: %w", id, err) + } + } + } + + return result, nil +} + +func getFilterRuleReferences(definition *settings.RuleDefinition) set.Set[string] { + result := set.New[string]() + + var addFilter func(filter settings.PatternFilter) + + addPatterns := func(patterns []settings.RulePattern) { + for _, pattern := range patterns { + for _, filter := range pattern.Filters { + addFilter(filter) + } + } + } + + addFilter = func(filter settings.PatternFilter) { + if filter.Detection != "" { + result.Add(filter.Detection) + } + + if filter.Not != nil { + addFilter(*filter.Not) + } + + for _, subFilter := range filter.Either { + addFilter(subFilter) + } + } + + addPatterns(definition.Patterns) + for _, auxiliaryDefinition := range definition.Auxiliary { + addPatterns(auxiliaryDefinition.Patterns) + } + + return result +} + +func getSanitizers(definition *settings.RuleDefinition) set.Set[string] { + result := set.New[string]() + + if definition.SanitizerRuleID != "" { + result.Add(definition.SanitizerRuleID) + } + + for _, auxiliaryDefinition := range definition.Auxiliary { + if auxiliaryDefinition.SanitizerRuleID != "" { + result.Add(auxiliaryDefinition.SanitizerRuleID) + } + } + + return result +} + +func getEnabledRules( + options flagtypes.RuleOptions, + definitions map[string]settings.RuleDefinition, + rules map[string]struct{}, +) map[string]struct{} { + enabledRules := make(map[string]struct{}) + + for ruleId := range rules { + enabledRules[ruleId] = struct{}{} + } + + var enableRule func(definition settings.RuleDefinition) + enableRule = func(definition settings.RuleDefinition) { + if definition.Disabled { + return + } + + id := definition.Metadata.ID + + if _, alreadyEnabled := enabledRules[id]; alreadyEnabled { + return + } + + enabledRules[id] = struct{}{} + + for _, dependencyID := range definition.Detectors { + enabledRules[dependencyID] = struct{}{} + } + + for _, importedRuleID := range definition.Imports { + if importedDefinition, exists := definitions[importedRuleID]; exists { + enableRule(importedDefinition) + } + } + } + + for _, definition := range definitions { + id := definition.Metadata.ID + + if len(options.OnlyRule) > 0 && !options.OnlyRule[id] { + continue + } + + if options.SkipRule[id] { + continue + } + + enableRule(definition) + } + + return enabledRules +} + +func BuildRules( + definitions map[string]settings.RuleDefinition, + enabledRules map[string]struct{}, +) map[string]*settings.Rule { + rules := make(map[string]*settings.Rule) + + for _, definition := range definitions { + id := definition.Metadata.ID + + if _, enabled := enabledRules[id]; !enabled { + continue + } + + ruleType := definition.Type + if len(ruleType) == 0 { + ruleType = defaultRuleType + } + + // build rule trigger + ruleTrigger := settings.RuleTrigger{ + MatchOn: settings.PRESENCE, + DataTypesRequired: false, + } + + if definition.Trigger != nil { + if definition.Trigger.MatchOn != nil { + ruleTrigger.MatchOn = *definition.Trigger.MatchOn + } + if definition.Trigger.DataTypesRequired != nil { + ruleTrigger.DataTypesRequired = *definition.Trigger.DataTypesRequired + } + if definition.Trigger.RequiredDetection != nil { + ruleTrigger.RequiredDetection = definition.Trigger.RequiredDetection + } + } + + isLocal := false + for _, rulePattern := range definition.Patterns { + if strings.Contains(rulePattern.Pattern, "$") { + isLocal = true + break + } + } + + rules[id] = &settings.Rule{ + Id: id, + Type: ruleType, + AssociatedRecipe: definition.Metadata.AssociatedRecipe, + Trigger: ruleTrigger, + IsLocal: isLocal, + SkipDataTypes: definition.SkipDataTypes, + OnlyDataTypes: definition.OnlyDataTypes, + Severity: definition.Severity, + Description: definition.Metadata.Description, + RemediationMessage: definition.Metadata.RemediationMessage, + Stored: definition.Stored, + Detectors: definition.Detectors, + Processors: definition.Processors, + AutoEncrytPrefix: definition.AutoEncrytPrefix, + CWEIDs: definition.Metadata.CWEIDs, + Languages: definition.Languages, + ParamParenting: definition.ParamParenting, + Patterns: definition.Patterns, + SanitizerRuleID: definition.SanitizerRuleID, + DocumentationUrl: definition.Metadata.DocumentationUrl, + HasDetailedContext: definition.HasDetailedContext, + DependencyCheck: definition.DependencyCheck, + Dependency: definition.Dependency, + } + + for _, auxiliaryDefinition := range definition.Auxiliary { + rules[auxiliaryDefinition.Id] = &settings.Rule{ + Id: auxiliaryDefinition.Id, + Type: defaultAuxiliaryRuleType, + Languages: definition.Languages, + ParamParenting: auxiliaryDefinition.ParamParenting, + Patterns: auxiliaryDefinition.Patterns, + SanitizerRuleID: auxiliaryDefinition.SanitizerRuleID, + Stored: auxiliaryDefinition.Stored, + IsAuxilary: true, + } + } + } + + return rules +} diff --git a/internal/commands/process/settings/rules/validation.go b/internal/commands/process/settings/rules/validation.go new file mode 100644 index 000000000..14ed93289 --- /dev/null +++ b/internal/commands/process/settings/rules/validation.go @@ -0,0 +1,204 @@ +package rules + +import ( + "bytes" + "fmt" + "net/http" + "path/filepath" + "sort" + "strings" + + "github.com/rs/zerolog/log" + "github.com/xeipuuv/gojsonschema" + "sigs.k8s.io/yaml" + + "github.com/bearer/bearer/internal/commands/process/settings" + flagtypes "github.com/bearer/bearer/internal/flag/types" + "github.com/bearer/bearer/internal/report/customdetectors" + "github.com/bearer/bearer/internal/util/output" + "github.com/bearer/bearer/internal/util/set" +) + +const SCHEMA_URL = "https://raw.githubusercontent.com/Bearer/bearer-rules/main/scripts/rule_schema.json" + +var builtinRuleIDs = []string{ + "datatype", + "insecure_url", + "string_literal", +} + +func validateCustomRuleSchema(entry []byte, filename string) string { + validationStr := &strings.Builder{} + validationStr.WriteString(fmt.Sprintf("Failed to load %s\nValidating against %s\n\n", filename, SCHEMA_URL)) + schema, err := loadSchema(SCHEMA_URL) + if err != nil { + validationStr.WriteString("Could not load schema to validate") + return validationStr.String() + } + + jsonData, err := yaml.YAMLToJSON(entry) + if err != nil { + validationStr.WriteString("File format is invalid") + return validationStr.String() + } + + result, err := validateData(jsonData, schema) + if err != nil { + validationStr.WriteString("Could not apply validation") + return validationStr.String() + } + + if result.Valid() { + validationStr.WriteString("Format of appears valid but could not be loaded") + } else { + validationStr.WriteString(fmt.Sprintf("%s validation issues found:\n", filename)) + for _, desc := range result.Errors() { + validationStr.WriteString(fmt.Sprintf("- %s\n", desc)) + } + fmt.Print("\n") + } + return validationStr.String() +} + +func loadSchema(url string) (*gojsonschema.Schema, error) { + response, err := http.Get(url) + if err != nil { + return nil, err + } + defer response.Body.Close() + + var buf bytes.Buffer + _, err = buf.ReadFrom(response.Body) + if err != nil { + return nil, err + } + + loader := gojsonschema.NewStringLoader(buf.String()) + return gojsonschema.NewSchema(loader) +} + +func validateData(data []byte, schema *gojsonschema.Schema) (*gojsonschema.Result, error) { + loader := gojsonschema.NewStringLoader(string(data)) + return schema.Validate(loader) +} + +func validateCustomRuleDefinition(allDefinitions map[string]settings.RuleDefinition, definition *settings.RuleDefinition) bool { + metadata := definition.Metadata + + valid := true + fail := func(message string) { + valid = false + log.Debug().Msgf("%s: %s", metadata.ID, message) + } + + visibleRuleIDs := set.New[string]() + visibleRuleIDs.Add(metadata.ID) + visibleRuleIDs.AddAll(builtinRuleIDs) + + for _, importedID := range definition.Imports { + visibleRuleIDs.Add(importedID) + + importedDefinition, exists := allDefinitions[importedID] + + if !exists { + fail(fmt.Sprintf("import of unknown rule '%s'", importedID)) + continue + } + + if importedDefinition.Type != customdetectors.TypeShared { + fail(fmt.Sprintf("imported rule '%s' is not of type 'shared'", importedID)) + } + } + + for _, auxiliaryDefinition := range definition.Auxiliary { + visibleRuleIDs.Add(auxiliaryDefinition.Id) + } + + for _, filterRuleID := range getFilterRuleReferences(definition).Items() { + if !visibleRuleIDs.Has(filterRuleID) { + fail(fmt.Sprintf("filter references invalid or non-imported rule '%s'", filterRuleID)) + } + } + + for _, sanitizerRuleID := range getSanitizers(definition).Items() { + if !visibleRuleIDs.Has(sanitizerRuleID) { + fail(fmt.Sprintf("sanitizer references invalid or non-imported rule '%s'", sanitizerRuleID)) + } + } + + if metadata.ID == "" { + fail("metadata.id must be specified") + } + + if definition.Type == customdetectors.TypeShared { + metadata := definition.Metadata + if metadata != nil { + if metadata.CWEIDs != nil { + fail("cwe ids cannot be specified for a shared rule") + } + + if metadata.RemediationMessage != "" { + fail("remediation message cannot be specified for a shared rule") + } + } + + if definition.Severity != "" { + fail("severity cannot be specified for a shared rule") + } + } + + if !valid { + log.Debug().Msgf("%s ignored due to validation errors", metadata.ID) + } + + return valid +} + +func validateRuleOptionIDs( + options flagtypes.RuleOptions, + definitions map[string]settings.RuleDefinition, + builtInDefinitions map[string]settings.RuleDefinition, +) error { + var invalidRuleIDs []string + + for id := range options.OnlyRule { + _, existsInDefinition := definitions[id] + _, existsInBuiltInDefinition := builtInDefinitions[id] + + if !existsInBuiltInDefinition && !existsInDefinition { + invalidRuleIDs = append(invalidRuleIDs, id) + } + } + var invalidSkipRuleIDs []string + for id := range options.SkipRule { + _, existsInDefinition := definitions[id] + _, existsInBuiltInDefinition := builtInDefinitions[id] + + if !existsInBuiltInDefinition && !existsInDefinition { + invalidSkipRuleIDs = append(invalidSkipRuleIDs, id) + } + } + + if len(invalidSkipRuleIDs) > 0 { + sort.Strings(invalidSkipRuleIDs) + output.StdErrLog(fmt.Sprintf("Warning: rule IDs %s given to be skipped but were not found", strings.Join(invalidSkipRuleIDs, ","))) + } + if len(invalidRuleIDs) > 0 { + return fmt.Errorf("invalid rule IDs in only option: %s", strings.Join(invalidRuleIDs, ",")) + } + + return nil +} + +func isRuleFile(path string) bool { + if strings.Contains(path, ".snapshots") { + return false + } + + ext := filepath.Ext(path) + if ext != ".yaml" && ext != ".yml" { + return false + } + + return strings.Contains(path, "/") +} diff --git a/internal/commands/process/settings/settings.go b/internal/commands/process/settings/settings.go index 2f383a76a..0ff0f16cf 100644 --- a/internal/commands/process/settings/settings.go +++ b/internal/commands/process/settings/settings.go @@ -1,21 +1,12 @@ package settings import ( - "embed" - "errors" - "fmt" "time" - "golang.org/x/exp/slices" - "gopkg.in/yaml.v3" - "github.com/bearer/bearer/api" - "github.com/bearer/bearer/internal/flag" flagtypes "github.com/bearer/bearer/internal/flag/types" - "github.com/bearer/bearer/internal/util/ignore" ignoretypes "github.com/bearer/bearer/internal/util/ignore/types" "github.com/bearer/bearer/internal/util/rego" - "github.com/bearer/bearer/internal/version_check" globaltypes "github.com/bearer/bearer/internal/types" ) @@ -68,8 +59,23 @@ type Config struct { IgnoreGit bool `mapstructure:"ignore_git" json:"ignore_git" yaml:"ignore_git"` } +type Processor struct { + Query string `mapstructure:"query" json:"query" yaml:"query"` + Modules Modules `mapstructure:"modules" json:"modules" yaml:"modules"` +} + type Modules []*PolicyModule +func (modules Modules) ToRegoModules() (output []rego.Module) { + for _, module := range modules { + output = append(output, rego.Module{ + Name: module.Name, + Content: module.Content, + }) + } + return +} + type Policy struct { Type string `mapstructure:"type" json:"type" yaml:"type"` Query string `mapstructure:"query" json:"query" yaml:"query"` @@ -102,13 +108,6 @@ const ( DefaultScope = NESTED_SCOPE ) -type LoadRulesResult struct { - BuiltInRules map[string]*Rule - Rules map[string]*Rule - CacheUsed bool - BearerRulesVersion string -} - type RuleTrigger struct { MatchOn MatchOn `mapstructure:"match_on" json:"match_on" yaml:"match_on"` DataTypesRequired bool `mapstructure:"data_types_required" json:"data_types_required" yaml:"data_types_required"` @@ -137,7 +136,7 @@ type RuleDefinition struct { Imports []string `mapstructure:"imports" json:"imports" yaml:"imports"` ParamParenting bool `mapstructure:"param_parenting" json:"param_parenting" yaml:"param_parenting"` Patterns []RulePattern `mapstructure:"patterns" json:"patterns" yaml:"patterns"` - SanitizerRuleID string `mapstructure:"sanitizer" json:"sanitizer" yaml:"sanitizer"` + SanitizerRuleID string `mapstructure:"sanitizer" json:"sanitizer,omitempty" yaml:"sanitizer,omitempty"` Stored bool `mapstructure:"stored" json:"stored" yaml:"stored"` Detectors []string `mapstructure:"detectors" json:"detectors,omitempty" yaml:"detectors,omitempty"` Processors []string `mapstructure:"processors" json:"processors,omitempty" yaml:"processors,omitempty"` @@ -152,6 +151,7 @@ type RuleDefinition struct { Auxiliary []Auxiliary `mapstructure:"auxiliary" json:"auxiliary" yaml:"auxiliary"` DependencyCheck bool `mapstructure:"dependency_check" json:"dependency_check" yaml:"dependency_check"` Dependency *Dependency `mapstructure:"dependency" json:"dependency" yaml:"dependency"` + Text string `mapstructure:"-" json:"-" yaml:"-"` } type Dependency struct { @@ -165,7 +165,7 @@ type Auxiliary struct { Type string `mapstructure:"type" json:"type" yaml:"type"` Languages []string `mapstructure:"languages" json:"languages" yaml:"languages"` Patterns []RulePattern `mapstructure:"patterns" json:"patterns" yaml:"patterns"` - SanitizerRuleID string `mapstructure:"sanitizer" json:"sanitizer" yaml:"sanitizer"` + SanitizerRuleID string `mapstructure:"sanitizer" json:"sanitizer,omitempty" yaml:"sanitizer,omitempty"` RootSingularize bool `mapstructure:"root_singularize" yaml:"root_singularize" ` RootLowercase bool `mapstructure:"root_lowercase" yaml:"root_lowercase"` @@ -219,168 +219,36 @@ type RuleReferenceImport struct { } type PatternFilter struct { - Not *PatternFilter `mapstructure:"not" json:"not" yaml:"not"` - Either []PatternFilter `mapstructure:"either" json:"either" yaml:"either"` - Variable string `mapstructure:"variable" json:"variable" yaml:"variable"` - Detection string `mapstructure:"detection" json:"detection" yaml:"detection"` - Scope RuleReferenceScope `mapstructure:"scope" json:"scope" yaml:"scope"` - Filters []PatternFilter `mapstructure:"filters" json:"filters" yaml:"filters"` - Imports []RuleReferenceImport `mapstructure:"imports" json:"imports" yaml:"imports"` + Not *PatternFilter `mapstructure:"not" json:"not,omitempty" yaml:"not,omitempty"` + Either []PatternFilter `mapstructure:"either" json:"either,omitempty" yaml:"either,omitempty"` + Variable string `mapstructure:"variable" json:"variable,omitempty" yaml:"variable,omitempty"` + Type string `mapstructure:"type" json:"type,omitempty" yaml:"type,omitempty"` + StaticType string `mapstructure:"static_type" json:"static_type,omitempty" yaml:"static_type,omitempty"` + Detection string `mapstructure:"detection" json:"detection,omitempty" yaml:"detection,omitempty"` + Scope RuleReferenceScope `mapstructure:"scope" json:"scope,omitempty" yaml:"scope,omitempty"` + IsSource bool `mapstructure:"is_source" json:"is_source" yaml:"is_source"` + Filters []PatternFilter `mapstructure:"filters" json:"filters,omitempty" yaml:"filters,omitempty"` + Imports []RuleReferenceImport `mapstructure:"imports" json:"imports,omitempty" yaml:"imports,omitempty"` // Contains is deprecated in favour of Scope - Contains *bool `mapstructure:"contains" json:"contains" yaml:"contains"` - Regex *Regexp `mapstructure:"regex" json:"regex" yaml:"regex"` - Values []string `mapstructure:"values" json:"values" yaml:"values"` - LengthLessThan *int `mapstructure:"length_less_than" json:"length_less_than" yaml:"length_less_than"` - LessThan *int `mapstructure:"less_than" json:"less_than" yaml:"less_than"` - LessThanOrEqual *int `mapstructure:"less_than_or_equal" json:"less_than_or_equal" yaml:"less_than_or_equal"` - GreaterThan *int `mapstructure:"greater_than" json:"greater_than" yaml:"greater_than"` - GreaterThanOrEqual *int `mapstructure:"greater_than_or_equal" json:"greater_than_or_equal" yaml:"greater_than_or_equal"` - StringRegex *Regexp `mapstructure:"string_regex" json:"string_regex" yaml:"string_regex"` - EntropyGreaterThan *float64 `mapstructure:"entropy_greater_than" json:"entropy_greater_than" yaml:"entropy_greater_than"` - FilenameRegex *Regexp `mapstructure:"filename_regex" json:"filename_regex" yaml:"filename_regex"` + Contains *bool `mapstructure:"contains" json:"contains,omitempty" yaml:"contains,omitempty"` + Regex *Regexp `mapstructure:"regex" json:"regex,omitempty" yaml:"regex,omitempty"` + Values []string `mapstructure:"values" json:"values,omitempty" yaml:"values,omitempty"` + LengthLessThan *int `mapstructure:"length_less_than" json:"length_less_than,omitempty" yaml:"length_less_than,omitempty"` + LessThan *int `mapstructure:"less_than" json:"less_than,omitempty" yaml:"less_than,omitempty"` + LessThanOrEqual *int `mapstructure:"less_than_or_equal" json:"less_than_or_equal,omitempty" yaml:"less_than_or_equal,omitempty"` + GreaterThan *int `mapstructure:"greater_than" json:"greater_than,omitempty" yaml:"greater_than,omitempty"` + GreaterThanOrEqual *int `mapstructure:"greater_than_or_equal" json:"greater_than_or_equal,omitempty" yaml:"greater_than_or_equal,omitempty"` + StringRegex *Regexp `mapstructure:"string_regex" json:"string_regex,omitempty" yaml:"string_regex,omitempty"` + EntropyGreaterThan *float64 `mapstructure:"entropy_greater_than" json:"entropy_greater_than,omitempty" yaml:"entropy_greater_than,omitempty"` + FilenameRegex *Regexp `mapstructure:"filename_regex" json:"filename_regex,omitempty" yaml:"filename_regex,omitempty"` } type RulePattern struct { Pattern string `mapstructure:"pattern" json:"pattern" yaml:"pattern"` - Focus string `mapstructure:"focus" json:"focus" yaml:"focus"` + Focus string `mapstructure:"focus" json:"focus,omitempty" yaml:"focus,omitempty"` Filters []PatternFilter `mapstructure:"filters" json:"filters" yaml:"filters"` } -type Processor struct { - Query string `mapstructure:"query" json:"query" yaml:"query"` - Modules Modules `mapstructure:"modules" json:"modules" yaml:"modules"` -} - -type MetaVar struct { - Input string `mapstructure:"input" json:"input" yaml:"input"` - Output int `mapstructure:"output" json:"output" yaml:"output"` - Regex string `mapstructure:"regex" json:"regex" yaml:"regex"` -} - -//go:embed policies.yml -var defaultPolicies []byte - -//go:embed built_in_rules/* -var buildInRulesFs embed.FS - -//go:embed policies/* -var policiesFs embed.FS - -//go:embed processors/* -var processorsFs embed.FS - -func (rule *Rule) PolicyType() bool { - return rule.Type == "risk" -} - -func (rule *Rule) GetSeverity() string { - if rule.Severity == "" { - return globaltypes.LevelLow - } - - return rule.Severity -} - -func (rule *Rule) Language() string { - if rule.Languages == nil { - return "secret" - } - - switch rule.Languages[0] { - case "java": - return "Java" - case "javascript": - return "JavaScript" - case "ruby": - return "Ruby" - case "sql": - return "SQL" - case "go": - return "Go" - case "php": - return "PHP" - case "python": - return "Python" - default: - return rule.Languages[0] - } -} - -func defaultWorkerOptions() WorkerOptions { - return WorkerOptions{ - Timeout: Timeout, - TimeoutFileMinimum: TimeoutFileMinimum, - TimeoutFileMaximum: TimeoutFileMaximum, - TimeoutFileBytesPerSecond: TimeoutFileBytesPerSecond, - TimeoutWorkerOnline: TimeoutWorkerOnline, - FileSizeMaximum: FileSizeMaximum, - ExistingWorker: ExistingWorker, - } -} - -func FromOptions(opts flagtypes.Options, versionMeta *version_check.VersionMeta) (Config, error) { - policies, err := DefaultPolicies() - if err != nil { - return Config{}, err - } - workerOptions := defaultWorkerOptions() - result, err := loadRules( - opts.ExternalRuleDir, - opts.RuleOptions, - versionMeta, - opts.ScanOptions.Force, - ) - if err != nil { - return Config{}, err - } - - for key := range policies { - policy := policies[key] - - for _, module := range policy.Modules { - if module.Path != "" { - content, err := policiesFs.ReadFile(module.Path) - if err != nil { - return Config{}, err - } - module.Content = string(content) - } - } - } - - ignoredFingerprints, _, _, err := ignore.GetIgnoredFingerprints(opts.GeneralOptions.IgnoreFile, &opts.ScanOptions.Target) - if err != nil { - return Config{}, err - } - - config := Config{ - Client: opts.Client, - Worker: workerOptions, - Scan: opts.ScanOptions, - Report: opts.ReportOptions, - IgnoredFingerprints: ignoredFingerprints, - NoColor: opts.GeneralOptions.NoColor || opts.ReportOptions.Output != "", - DebugProfile: opts.GeneralOptions.DebugProfile, - Debug: opts.GeneralOptions.Debug, - LogLevel: opts.GeneralOptions.LogLevel, - IgnoreFile: opts.GeneralOptions.IgnoreFile, - IgnoreGit: opts.GeneralOptions.IgnoreGit, - Policies: policies, - Rules: result.Rules, - BuiltInRules: result.BuiltInRules, - CacheUsed: result.CacheUsed, - BearerRulesVersion: result.BearerRulesVersion, - } - - if config.Scan.Diff { - if !slices.Contains([]string{flag.ReportSecurity, flag.ReportSaaS}, config.Report.Report) { - return Config{}, errors.New("diff base branch is only supported for the security report") - } - } - - return config, nil -} - func (rulePattern *RulePattern) UnmarshalYAML(unmarshal func(interface{}) error) error { // Try to parse as a string var pattern string @@ -419,38 +287,20 @@ func (filter *PatternFilter) UnmarshalYAML(unmarshal func(interface{}) error) er return nil } -func DefaultPolicies() (map[string]*Policy, error) { - policies := make(map[string]*Policy) - var policy []*Policy - - err := yaml.Unmarshal(defaultPolicies, &policy) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal policy file %s", err) - } - - for _, policy := range policy { - policies[policy.Type] = policy - } - - return policies, nil +type MetaVar struct { + Input string `mapstructure:"input" json:"input" yaml:"input"` + Output int `mapstructure:"output" json:"output" yaml:"output"` + Regex string `mapstructure:"regex" json:"regex" yaml:"regex"` } -func ProcessorRegoModuleText(processorName string) (string, error) { - processorPath := fmt.Sprintf("processors/%s.rego", processorName) - data, err := processorsFs.ReadFile(processorPath) - if err != nil { - return "", err - } - - return string(data), nil +func (rule *Rule) PolicyType() bool { + return rule.Type == "risk" } -func (modules Modules) ToRegoModules() (output []rego.Module) { - for _, module := range modules { - output = append(output, rego.Module{ - Name: module.Name, - Content: module.Content, - }) +func (rule *Rule) GetSeverity() string { + if rule.Severity == "" { + return globaltypes.LevelLow } - return + + return rule.Severity } diff --git a/internal/commands/scan.go b/internal/commands/scan.go index a0eff5d7c..bd47ceba6 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -5,6 +5,7 @@ import ( "github.com/bearer/bearer/internal/commands/artifact" "github.com/bearer/bearer/internal/commands/debugprofile" + "github.com/bearer/bearer/internal/engine" "github.com/bearer/bearer/internal/flag" "github.com/bearer/bearer/internal/util/file" "github.com/bearer/bearer/internal/util/output" @@ -38,7 +39,7 @@ var ScanFlags = flag.Flags{ flag.GeneralFlagGroup, } -func NewScanCommand() *cobra.Command { +func NewScanCommand(engine engine.Engine) *cobra.Command { cmd := &cobra.Command{ Use: "scan [flags] ", Aliases: []string{"s"}, @@ -84,7 +85,7 @@ func NewScanCommand() *cobra.Command { cmd.SilenceUsage = true - err = artifact.Run(cmd.Context(), options) + err = artifact.Run(cmd.Context(), options, engine) debugprofile.Stop() return err }, diff --git a/internal/engine/engine.go b/internal/engine/engine.go new file mode 100644 index 000000000..4af7747fe --- /dev/null +++ b/internal/engine/engine.go @@ -0,0 +1,17 @@ +package engine + +import ( + "github.com/bearer/bearer/internal/commands/process/filelist/files" + "github.com/bearer/bearer/internal/commands/process/settings" + "github.com/bearer/bearer/internal/scanner/language" + "github.com/bearer/bearer/internal/scanner/stats" +) + +type Engine interface { + GetLanguages() []language.Language + GetLanguageById(id string) language.Language + Initialize(config *settings.Config) error + LoadRule(yamlDefinition string) error + Scan(stats *stats.Stats, reportPath, targetPath string, files []files.File) error + Close() +} diff --git a/internal/engine/implementation/implementation.go b/internal/engine/implementation/implementation.go new file mode 100644 index 000000000..9321ced8b --- /dev/null +++ b/internal/engine/implementation/implementation.go @@ -0,0 +1,68 @@ +package implementation + +import ( + "github.com/bearer/bearer/internal/commands/process/filelist/files" + "github.com/bearer/bearer/internal/commands/process/orchestrator" + "github.com/bearer/bearer/internal/commands/process/orchestrator/work" + "github.com/bearer/bearer/internal/commands/process/settings" + enginepkg "github.com/bearer/bearer/internal/engine" + "github.com/bearer/bearer/internal/scanner/language" + "github.com/bearer/bearer/internal/scanner/stats" +) + +type engine struct { + languages []language.Language + config *settings.Config + orchestrator *orchestrator.Orchestrator +} + +func New(languages []language.Language) enginepkg.Engine { + return &engine{languages: languages} +} + +func (engine *engine) GetLanguages() []language.Language { + return engine.languages +} + +func (engine *engine) GetLanguageById(id string) language.Language { + for _, language := range engine.languages { + if language.ID() == id { + return language + } + } + + return nil +} + +func (engine *engine) Initialize(config *settings.Config) error { + engine.config = config + + return nil +} + +func (engine *engine) LoadRule(yamlDefinition string) error { + return nil +} + +func (engine *engine) Scan( + stats *stats.Stats, + reportPath, + targetPath string, + files []files.File, +) error { + if engine.orchestrator == nil { + var err error + engine.orchestrator, err = orchestrator.New(work.Repository{Dir: targetPath}, engine.config, stats, len(files)) + if err != nil { + return err + } + } + + return engine.orchestrator.Scan(reportPath, files) +} + +func (engine *engine) Close() { + if engine.orchestrator != nil { + engine.orchestrator.Close() + } +} diff --git a/internal/languages/golang/golang.go b/internal/languages/golang/golang.go index ba2412322..8726054d1 100644 --- a/internal/languages/golang/golang.go +++ b/internal/languages/golang/golang.go @@ -34,6 +34,10 @@ func (*implementation) ID() string { return "go" } +func (*implementation) DisplayName() string { + return "Go" +} + func (*implementation) EnryLanguages() []string { return []string{"Go"} } diff --git a/internal/languages/java/java.go b/internal/languages/java/java.go index 4d0b67d82..b813f7e19 100644 --- a/internal/languages/java/java.go +++ b/internal/languages/java/java.go @@ -32,6 +32,10 @@ func (*implementation) ID() string { return "java" } +func (*implementation) DisplayName() string { + return "Java" +} + func (*implementation) EnryLanguages() []string { return []string{"Java"} } diff --git a/internal/languages/javascript/javascript.go b/internal/languages/javascript/javascript.go index e9b2033be..fcbc55d73 100644 --- a/internal/languages/javascript/javascript.go +++ b/internal/languages/javascript/javascript.go @@ -32,6 +32,10 @@ func (*implementation) ID() string { return "javascript" } +func (*implementation) DisplayName() string { + return "JavaScript" +} + func (*implementation) EnryLanguages() []string { return []string{"JavaScript", "TypeScript", "TSX"} } diff --git a/internal/languages/languages.go b/internal/languages/languages.go new file mode 100644 index 000000000..d8d9d647d --- /dev/null +++ b/internal/languages/languages.go @@ -0,0 +1,22 @@ +package languages + +import ( + "github.com/bearer/bearer/internal/languages/golang" + "github.com/bearer/bearer/internal/languages/java" + "github.com/bearer/bearer/internal/languages/javascript" + "github.com/bearer/bearer/internal/languages/php" + "github.com/bearer/bearer/internal/languages/python" + "github.com/bearer/bearer/internal/languages/ruby" + "github.com/bearer/bearer/internal/scanner/language" +) + +func Default() []language.Language { + return []language.Language{ + golang.Get(), + java.Get(), + javascript.Get(), + php.Get(), + python.Get(), + ruby.Get(), + } +} diff --git a/internal/languages/php/php.go b/internal/languages/php/php.go index 8acfb81bb..96efde1cb 100644 --- a/internal/languages/php/php.go +++ b/internal/languages/php/php.go @@ -32,6 +32,10 @@ func (*implementation) ID() string { return "php" } +func (*implementation) DisplayName() string { + return "PHP" +} + func (*implementation) EnryLanguages() []string { return []string{"PHP"} } diff --git a/internal/languages/python/python.go b/internal/languages/python/python.go index 9de500ca8..16bcc88b7 100644 --- a/internal/languages/python/python.go +++ b/internal/languages/python/python.go @@ -32,6 +32,10 @@ func (*implementation) ID() string { return "python" } +func (*implementation) DisplayName() string { + return "Python" +} + func (*implementation) EnryLanguages() []string { return []string{"Python"} } diff --git a/internal/languages/ruby/ruby.go b/internal/languages/ruby/ruby.go index cce31f392..5e6f898bc 100644 --- a/internal/languages/ruby/ruby.go +++ b/internal/languages/ruby/ruby.go @@ -32,6 +32,10 @@ func (*implementation) ID() string { return "ruby" } +func (*implementation) DisplayName() string { + return "Ruby" +} + func (*implementation) EnryLanguages() []string { return []string{"Ruby"} } diff --git a/internal/languages/testhelper/testhelper.go b/internal/languages/testhelper/testhelper.go index 1a201b79f..94ce016a1 100644 --- a/internal/languages/testhelper/testhelper.go +++ b/internal/languages/testhelper/testhelper.go @@ -18,7 +18,11 @@ import ( "github.com/bearer/bearer/internal/commands/process/orchestrator/work" "github.com/bearer/bearer/internal/commands/process/orchestrator/worker" "github.com/bearer/bearer/internal/commands/process/settings" + settingsloader "github.com/bearer/bearer/internal/commands/process/settings/loader" + "github.com/bearer/bearer/internal/commands/process/settings/rules" + engine "github.com/bearer/bearer/internal/engine/implementation" "github.com/bearer/bearer/internal/flag" + "github.com/bearer/bearer/internal/languages" "github.com/bearer/bearer/internal/report/output" "github.com/bearer/bearer/internal/types" util "github.com/bearer/bearer/internal/util/output" @@ -33,7 +37,7 @@ type Runner struct { func GetRunner(t *testing.T, ruleBytes []byte, lang string) *Runner { zerolog.SetGlobalLevel(zerolog.InfoLevel) - err := commands.ScanFlags.BindForConfigInit(commands.NewScanCommand()) + err := commands.ScanFlags.BindForConfigInit(commands.NewScanCommand(nil)) if err != nil { t.Fatalf("failed to bind flags: %s", err) } @@ -55,7 +59,9 @@ func GetRunner(t *testing.T, ruleBytes []byte, lang string) *Runner { Message: "", }, } - config, err := settings.FromOptions(configFlags, meta) + + engine := engine.New(languages.Default()) + config, err := settingsloader.FromOptions(configFlags, meta, engine) if err != nil { t.Fatalf("failed to generate default scan settings: %s", err) } @@ -83,14 +89,14 @@ func getRulesFromYaml(t *testing.T, ruleBytes []byte) map[string]*settings.Rule t.Fatalf("failed to unmarshal rule %s", err) } - rules := map[string]settings.RuleDefinition{ + definitions := map[string]settings.RuleDefinition{ ruleDefinition.Metadata.ID: ruleDefinition, } enabledRules := map[string]struct{}{ ruleDefinition.Metadata.ID: {}, } - return settings.BuildRules(rules, enabledRules) + return rules.BuildRules(definitions, enabledRules) } func (runner *Runner) RunTest(t *testing.T, testdataPath string, snapshotPath string) { diff --git a/internal/report/output/dataflow/datatypes/get_extras.go b/internal/report/output/dataflow/datatypes/get_extras.go index 9da121d9b..dc2211fc5 100644 --- a/internal/report/output/dataflow/datatypes/get_extras.go +++ b/internal/report/output/dataflow/datatypes/get_extras.go @@ -5,12 +5,14 @@ import ( "errors" "fmt" + "github.com/open-policy-agent/opa/rego" + "github.com/bearer/bearer/internal/commands/process/settings" + "github.com/bearer/bearer/internal/commands/process/settings/processors" "github.com/bearer/bearer/internal/report/detections" "github.com/bearer/bearer/internal/report/detectors" "github.com/bearer/bearer/internal/report/output/dataflow/types" regohelper "github.com/bearer/bearer/internal/util/rego" - "github.com/open-policy-agent/opa/rego" ) type processorInput struct { @@ -287,28 +289,13 @@ func (extras *extrasObj) Get(detection interface{}) *ExtraFields { return extras.data[detectionID] } -func processorModules(processorName string) (modules []regohelper.Module, err error) { - moduleText, err := settings.ProcessorRegoModuleText(processorName) - if err != nil { - return - } - - fullModuleName := fmt.Sprintf("bearer.%s", processorName) - modules = []regohelper.Module{{ - Name: fullModuleName, - Content: moduleText, - }} - - return -} - func runProcessor( processorName string, detections []any, targetDetections []any, rule *settings.Rule, ) (data map[string]*ExtraFields, err error) { - modules, err := processorModules(processorName) + modules, err := processors.Load(processorName) if err != nil { return } diff --git a/internal/report/output/output.go b/internal/report/output/output.go index bf9da13ca..bc061da08 100644 --- a/internal/report/output/output.go +++ b/internal/report/output/output.go @@ -11,6 +11,7 @@ import ( "github.com/bearer/bearer/internal/commands/process/gitrepository" "github.com/bearer/bearer/internal/commands/process/settings" + "github.com/bearer/bearer/internal/engine" "github.com/bearer/bearer/internal/flag" "github.com/bearer/bearer/internal/report/basebranchfindings" "github.com/bearer/bearer/internal/report/output/dataflow" @@ -103,6 +104,7 @@ func GetDataflow( func FormatOutput( reportData *types.ReportData, config settings.Config, + engine engine.Engine, goclocResult *gocloc.Result, startTime time.Time, endTime time.Time, @@ -114,7 +116,7 @@ func FormatOutput( case flag.ReportDataFlow: formatter = dataflow.NewFormatter(reportData, config) case flag.ReportSecurity: - formatter = security.NewFormatter(reportData, config, goclocResult, startTime, endTime) + formatter = security.NewFormatter(reportData, config, engine, goclocResult, startTime, endTime) case flag.ReportPrivacy: formatter = privacy.NewFormatter(reportData, config) case flag.ReportSaaS: diff --git a/internal/report/output/privacy/privacy.go b/internal/report/output/privacy/privacy.go index 6ed4ee316..67cf0669f 100644 --- a/internal/report/output/privacy/privacy.go +++ b/internal/report/output/privacy/privacy.go @@ -108,7 +108,7 @@ func AddReportData(reportData *outputtypes.ReportData, config settings.Config) e output.StdErrLog("Evaluating rules") } - bar := progressbar.GetProgressBar(len(config.Rules), config) + bar := progressbar.GetProgressBar(len(config.Rules), &config) subjectRuleFailures := make(map[string]RuleFailureSummary) thirdPartyRuleFailures := make(map[string]map[string]RuleFailureSummary) diff --git a/internal/report/output/privacy/privacy_test.go b/internal/report/output/privacy/privacy_test.go index af94697e1..1c1a8e637 100644 --- a/internal/report/output/privacy/privacy_test.go +++ b/internal/report/output/privacy/privacy_test.go @@ -6,7 +6,11 @@ import ( "github.com/bradleyjkemp/cupaloy" "github.com/bearer/bearer/internal/commands/process/settings" + settingsloader "github.com/bearer/bearer/internal/commands/process/settings/loader" + "github.com/bearer/bearer/internal/engine" + engineimpl "github.com/bearer/bearer/internal/engine/implementation" flagtypes "github.com/bearer/bearer/internal/flag/types" + "github.com/bearer/bearer/internal/languages" "github.com/bearer/bearer/internal/report/output/dataflow/types" "github.com/bearer/bearer/internal/report/output/privacy" "github.com/bearer/bearer/internal/report/output/testhelper" @@ -16,7 +20,8 @@ import ( ) func TestBuildCsvString(t *testing.T) { - config, err := generateConfig(flagtypes.ReportOptions{Report: "privacy"}) + engine := engineimpl.New(languages.Default()) + config, err := generateConfig(engine, flagtypes.ReportOptions{Report: "privacy"}) config.Rules = map[string]*settings.Rule{ "ruby_third_parties_sentry": testhelper.RubyThirdPartiesSentryRule(), } @@ -37,7 +42,8 @@ func TestBuildCsvString(t *testing.T) { } func TestAddReportData(t *testing.T) { - config, err := generateConfig(flagtypes.ReportOptions{Report: "privacy"}) + engine := engineimpl.New(languages.Default()) + config, err := generateConfig(engine, flagtypes.ReportOptions{Report: "privacy"}) config.Rules = map[string]*settings.Rule{ "ruby_third_parties_sentry": testhelper.RubyThirdPartiesSentryRule(), } @@ -56,7 +62,7 @@ func TestAddReportData(t *testing.T) { cupaloy.SnapshotT(t, output.PrivacyReport) } -func generateConfig(reportOptions flagtypes.ReportOptions) (settings.Config, error) { +func generateConfig(engine engine.Engine, reportOptions flagtypes.ReportOptions) (settings.Config, error) { opts := flagtypes.Options{ ScanOptions: flagtypes.ScanOptions{ Scanner: []string{"sast"}, @@ -75,7 +81,8 @@ func generateConfig(reportOptions flagtypes.ReportOptions) (settings.Config, err Message: "", }, } - return settings.FromOptions(opts, meta) + + return settingsloader.FromOptions(opts, meta, engine) } func dummyDataflow() *outputtypes.DataFlow { diff --git a/internal/report/output/security/formatter.go b/internal/report/output/security/formatter.go index 91f1561cf..6f5440ac1 100644 --- a/internal/report/output/security/formatter.go +++ b/internal/report/output/security/formatter.go @@ -8,6 +8,7 @@ import ( "github.com/bearer/bearer/cmd/bearer/build" "github.com/bearer/bearer/internal/commands/process/settings" + "github.com/bearer/bearer/internal/engine" "github.com/bearer/bearer/internal/flag" "github.com/bearer/bearer/internal/report/output/gitlab" "github.com/bearer/bearer/internal/report/output/html" @@ -20,6 +21,7 @@ import ( type Formatter struct { ReportData *outputtypes.ReportData Config settings.Config + engine engine.Engine GoclocResult *gocloc.Result StartTime time.Time EndTime time.Time @@ -32,10 +34,18 @@ type JsonV2Output struct { Expected ExpectedDetections `json:"expected_findings,omitempty" yaml:"expected_findings,omitempty"` } -func NewFormatter(reportData *outputtypes.ReportData, config settings.Config, goclocResult *gocloc.Result, startTime time.Time, endTime time.Time) *Formatter { +func NewFormatter( + reportData *outputtypes.ReportData, + config settings.Config, + engine engine.Engine, + goclocResult *gocloc.Result, + startTime time.Time, + endTime time.Time, +) *Formatter { return &Formatter{ ReportData: reportData, Config: config, + engine: engine, GoclocResult: goclocResult, StartTime: startTime, EndTime: endTime, @@ -45,7 +55,7 @@ func NewFormatter(reportData *outputtypes.ReportData, config settings.Config, go func (f Formatter) Format(format string) (output string, err error) { switch format { case flag.FormatEmpty: - output = BuildReportString(f.ReportData, f.Config, f.GoclocResult).String() + output = BuildReportString(f.ReportData, f.Config, f.engine, f.GoclocResult).String() case flag.FormatSarif: sarifContent, sarifErr := sarif.ReportSarif(f.ReportData.FindingsBySeverity, f.Config.Rules) if sarifErr != nil { diff --git a/internal/report/output/security/security.go b/internal/report/output/security/security.go index ddb0359eb..45e7a8f48 100644 --- a/internal/report/output/security/security.go +++ b/internal/report/output/security/security.go @@ -17,6 +17,7 @@ import ( "github.com/bearer/bearer/internal/classification/db" "github.com/bearer/bearer/internal/commands/process/settings" + "github.com/bearer/bearer/internal/engine" "github.com/bearer/bearer/internal/report/basebranchfindings" globaltypes "github.com/bearer/bearer/internal/types" "github.com/bearer/bearer/internal/util/file" @@ -154,7 +155,7 @@ func evaluateRules( var bar *progressbar.ProgressBar if !builtIn { - bar = bearerprogressbar.GetProgressBar(len(rules), config) + bar = bearerprogressbar.GetProgressBar(len(rules), &config) } var fingerprints []string @@ -392,7 +393,12 @@ func getExtract(rawCodeExtract []file.Line) string { return strings.Join(parts, "\n") } -func BuildReportString(reportData *outputtypes.ReportData, config settings.Config, lineOfCodeOutput *gocloc.Result) *strings.Builder { +func BuildReportString( + reportData *outputtypes.ReportData, + config settings.Config, + engine engine.Engine, + lineOfCodeOutput *gocloc.Result, +) *strings.Builder { reportStr := &strings.Builder{} if len(reportData.Files) == 0 { @@ -414,6 +420,7 @@ func BuildReportString(reportData *outputtypes.ReportData, config settings.Confi rulesAvailableCount := writeRuleListToString( reportStr, + engine, config.Rules, config.BuiltInRules, reportData.Dataflow.Dependencies, @@ -492,14 +499,15 @@ func writeStatsToString( func writeRuleListToString( reportStr *strings.Builder, + engine engine.Engine, rules map[string]*settings.Rule, builtInRules map[string]*settings.Rule, reportedDependencies []dataflowtypes.Dependency, languages map[string]*gocloc.Language, config settings.Config, ) int { - ruleCountPerLang, totalRuleCount, defaultRulesUsed := countRules(rules, languages, config, false) - builtInRuleCountPerLang, totalBuiltInRuleCount, builtInRulesUsed := countRules(builtInRules, languages, config, true) + ruleCountPerLang, totalRuleCount, defaultRulesUsed := countRules(engine, rules, languages, config, false) + builtInRuleCountPerLang, totalBuiltInRuleCount, builtInRulesUsed := countRules(engine, builtInRules, languages, config, true) // combine default and built-in rules per lang for _, lang := range maps.Keys(builtInRuleCountPerLang) { @@ -599,6 +607,7 @@ func getLanguagePairs(languages map[string]*gocloc.Language) []*gocloc.Language } func countRules( + engine engine.Engine, rules map[string]*settings.Rule, languages map[string]*gocloc.Language, config settings.Config, @@ -619,13 +628,15 @@ func countRules( var shouldCount bool - if rule.Language() == "secret" { + language := getLanguageDisplayName(engine, rule) + + if language == "secret" { shouldCount = slices.Contains(config.Scan.Scanner, "secrets") } else if slices.Contains(config.Scan.Scanner, "sast") { - if rule.Language() == "JavaScript" { + if language == "JavaScript" { shouldCount = languages["JavaScript"] != nil || languages["TypeScript"] != nil } else { - shouldCount = languages[rule.Language()] != nil + shouldCount = languages[language] != nil } } @@ -637,28 +648,28 @@ func countRules( totalRuleCount += 1 defaultRule := strings.HasPrefix(rule.DocumentationUrl, "https://docs.bearer.com") || builtIn - if ruleCount, ok := ruleCountPerLang[rule.Language()]; ok { + if ruleCount, ok := ruleCountPerLang[language]; ok { if defaultRule { if !defaultRulesUsed { defaultRulesUsed = true } ruleCount.DefaultRuleCount += 1 - ruleCountPerLang[rule.Language()] = ruleCount + ruleCountPerLang[language] = ruleCount } else { ruleCount.CustomRuleCount += 1 - ruleCountPerLang[rule.Language()] = ruleCount + ruleCountPerLang[language] = ruleCount } } else { if defaultRule { if !defaultRulesUsed { defaultRulesUsed = true } - ruleCountPerLang[rule.Language()] = RuleCounter{ + ruleCountPerLang[language] = RuleCounter{ CustomRuleCount: 0, DefaultRuleCount: 1, } } else { - ruleCountPerLang[rule.Language()] = RuleCounter{ + ruleCountPerLang[language] = RuleCounter{ CustomRuleCount: 1, DefaultRuleCount: 0, } @@ -860,3 +871,16 @@ func codeExtract(filename string, Source types.Source, Sink types.Sink) []file.L return code } + +func getLanguageDisplayName(engine engine.Engine, rule *settings.Rule) string { + if rule.Languages == nil { + return "secret" + } + + language := engine.GetLanguageById(rule.Languages[0]) + if language == nil { + return rule.Languages[0] + } + + return language.DisplayName() +} diff --git a/internal/report/output/security/security_test.go b/internal/report/output/security/security_test.go index 1434a2732..d921710da 100644 --- a/internal/report/output/security/security_test.go +++ b/internal/report/output/security/security_test.go @@ -9,8 +9,12 @@ import ( "github.com/bearer/bearer/internal/commands/process/filelist/files" "github.com/bearer/bearer/internal/commands/process/settings" + settingsloader "github.com/bearer/bearer/internal/commands/process/settings/loader" + "github.com/bearer/bearer/internal/engine" + engineimpl "github.com/bearer/bearer/internal/engine/implementation" flagtypes "github.com/bearer/bearer/internal/flag/types" "github.com/bearer/bearer/internal/git" + "github.com/bearer/bearer/internal/languages" "github.com/bearer/bearer/internal/report/basebranchfindings" "github.com/bearer/bearer/internal/report/schema" globaltypes "github.com/bearer/bearer/internal/types" @@ -24,7 +28,8 @@ import ( ) func TestBuildReportString(t *testing.T) { - config, err := generateConfig(flagtypes.ReportOptions{Report: "security"}) + engine := engineimpl.New(languages.Default()) + config, err := generateConfig(engine, flagtypes.ReportOptions{Report: "security"}) // set rule version config.BearerRulesVersion = "TEST" @@ -53,12 +58,13 @@ func TestBuildReportString(t *testing.T) { MaxPathLength: 0, } - stringBuilder := security.BuildReportString(data, config, &dummyGoclocResult) + stringBuilder := security.BuildReportString(data, config, engine, &dummyGoclocResult) cupaloy.SnapshotT(t, stringBuilder.String()) } func TestNoRulesBuildReportString(t *testing.T) { - config, err := generateConfig(flagtypes.ReportOptions{Report: "security"}) + engine := engineimpl.New(languages.Default()) + config, err := generateConfig(engine, flagtypes.ReportOptions{Report: "security"}) // set rule version config.BearerRulesVersion = "TEST" config.Rules = map[string]*settings.Rule{} @@ -82,12 +88,13 @@ func TestNoRulesBuildReportString(t *testing.T) { MaxPathLength: 0, } - stringBuilder := security.BuildReportString(output, config, &dummyGoclocResult) + stringBuilder := security.BuildReportString(output, config, engine, &dummyGoclocResult) cupaloy.SnapshotT(t, stringBuilder.String()) } func TestAddReportData(t *testing.T) { - config, err := generateConfig(flagtypes.ReportOptions{Report: "security"}) + engine := engineimpl.New(languages.Default()) + config, err := generateConfig(engine, flagtypes.ReportOptions{Report: "security"}) config.Rules = map[string]*settings.Rule{ "ruby_lang_ssl_verification": testhelper.RubyLangSSLVerificationRule(), @@ -109,10 +116,11 @@ func TestAddReportData(t *testing.T) { } func TestAddReportDataWithSeverity(t *testing.T) { + engine := engineimpl.New(languages.Default()) severity := set.New[string]() severity.Add(globaltypes.LevelCritical) - config, err := generateConfig(flagtypes.ReportOptions{ + config, err := generateConfig(engine, flagtypes.ReportOptions{ Report: "security", Severity: severity, }) @@ -166,6 +174,7 @@ func TestAddReportDataWithFailOnSeverity(t *testing.T) { }, } { t.Run(test.FailOnSeverity, func(tt *testing.T) { + engine := engineimpl.New(languages.Default()) failOnSeverity := set.New[string]() failOnSeverity.Add(test.FailOnSeverity) @@ -175,7 +184,7 @@ func TestAddReportDataWithFailOnSeverity(t *testing.T) { severity.Add(test.Severity) } - config, err := generateConfig(flagtypes.ReportOptions{ + config, err := generateConfig(engine, flagtypes.ReportOptions{ Report: "security", Severity: severity, FailOnSeverity: failOnSeverity, @@ -201,7 +210,8 @@ func TestAddReportDataWithFailOnSeverity(t *testing.T) { } func TestFingerprintIsStableWithBaseBranchFindings(t *testing.T) { - config, err := generateConfig(flagtypes.ReportOptions{Report: "security"}) + engine := engineimpl.New(languages.Default()) + config, err := generateConfig(engine, flagtypes.ReportOptions{Report: "security"}) if err != nil { t.Fatalf("failed to generate config:%s", err) } @@ -291,7 +301,7 @@ func TestFingerprintIsStableWithBaseBranchFindings(t *testing.T) { assert.Equal(t, fullScanFinding.Fingerprint, diffFinding.Fingerprint) } -func generateConfig(reportOptions flagtypes.ReportOptions) (settings.Config, error) { +func generateConfig(engine engine.Engine, reportOptions flagtypes.ReportOptions) (settings.Config, error) { if reportOptions.Severity == nil { reportOptions.Severity = set.New[string]() reportOptions.Severity.AddAll(globaltypes.Severities) @@ -323,7 +333,8 @@ func generateConfig(reportOptions flagtypes.ReportOptions) (settings.Config, err Message: "", }, } - return settings.FromOptions(opts, meta) + + return settingsloader.FromOptions(opts, meta, engine) } func dummyDataflowData() *outputtypes.ReportData { diff --git a/internal/scanner/language/language.go b/internal/scanner/language/language.go index 3992eae98..fa96fed43 100644 --- a/internal/scanner/language/language.go +++ b/internal/scanner/language/language.go @@ -11,6 +11,7 @@ import ( type Language interface { ID() string + DisplayName() string EnryLanguages() []string NewBuiltInDetectors(schemaClassifier *schema.Classifier, querySet *query.Set) []detectortypes.Detector SitterLanguage() *sitter.Language diff --git a/internal/util/progressbar/progressbar.go b/internal/util/progressbar/progressbar.go index 99cfb6853..e78a19ff7 100644 --- a/internal/util/progressbar/progressbar.go +++ b/internal/util/progressbar/progressbar.go @@ -6,7 +6,7 @@ import ( "github.com/schollz/progressbar/v3" ) -func GetProgressBar(filesLength int, config settings.Config) *progressbar.ProgressBar { +func GetProgressBar(filesLength int, config *settings.Config) *progressbar.ProgressBar { hideProgress := config.Scan.HideProgressBar || config.Scan.Quiet || config.Debug return progressbar.NewOptions(filesLength, progressbar.OptionSetVisibility(!hideProgress), diff --git a/scripts/gen-doc-yaml.go b/scripts/gen-doc-yaml.go index ea611fe14..ce791b571 100644 --- a/scripts/gen-doc-yaml.go +++ b/scripts/gen-doc-yaml.go @@ -74,7 +74,7 @@ func main() { os.Exit(1) } - cmd := commands.NewApp(build.Version, build.CommitSHA) + cmd := commands.NewApp(build.Version, build.CommitSHA, nil) err := writeDocs( cmd, dir,