chore: make engine pluggable (#1615)

.envrc.example

@@ -5,7 +5,7 @@ export GOOGLE_APP=''
 export REDIS_INIT="true"
 export GOOGLE_MAX_ATTEMPT="5"
 export BEARER_EXECUTABLE_PATH="./bearer"
-export GITHUB_WORKSPACE="/path/to/bearer/project"
+export GITHUB_WORKSPACE="$PWD"
 export SCAN_DIR=/Users/username/OWASP
 export BEARER_DISABLE_VERSION_CHECK=true
 export BEARER_DISABLE_DEFAULT_RULES=true

.gitattributes

@@ -1,2 +1,2 @@
-internal/parser/sitter/**/*.c linguist-generated
-internal/parser/sitter/**/*.cc linguist-generated
+pkg/parser/sitter/**/*.c linguist-generated
+pkg/parser/sitter/**/*.cc linguist-generated

.github/workflows/test.yml

@@ -24,4 +24,4 @@ jobs:
         with:
           go-version: 1.21
       - name: Run package tests
-        run: go test -v ./internal/...
+        run: go test -v ./pkg/...

.gitignore

@@ -19,4 +19,4 @@ bearer
 # docs
 docs/yarn.lock
 
-/internal/parser/sitter/config_variables/build
+/pkg/parser/sitter/config_variables/build

cmd/bearer/bearer.go

@@ -1,16 +1,10 @@
 package main
 
 import (
-	"os"
-
 	"github.com/bearer/bearer/cmd/bearer/build"
-	"github.com/bearer/bearer/internal/commands"
+	"github.com/bearer/bearer/external/run"
 )
 
 func main() {
-	app := commands.NewApp(build.Version, build.CommitSHA)
-	if err := app.Execute(); err != nil {
-		// error messages are printed by the framework
-		os.Exit(1)
-	}
+	run.Run(build.Version, build.CommitSHA, run.NewEngine(run.DefaultLanguages()))
 }

docs/_data/datatypes.js

@@ -69,12 +69,10 @@ function sortData(typesFile, catsFile, groupsFile) {
 }
 // example();
 module.exports = async function () {
-  const dataTypes = await fetchData("../internal/classification/db/data_types/")
-  const dataCats = await fetchData(
-    "../internal/classification/db/data_categories/",
-  )
+  const dataTypes = await fetchData("../pkg/classification/db/data_types/")
+  const dataCats = await fetchData("../pkg/classification/db/data_categories/")
   const groupings = await fetchFile(
-    "../internal/classification/db/category_grouping.json",
+    "../pkg/classification/db/category_grouping.json",
   )
   return sortData(dataTypes, dataCats, groupings)
 }

docs/_data/recipes.js

@@ -15,13 +15,13 @@ async function fetchData(dir) {
       return {
         ...data,
         id: path.basename(file, ".json"),
-        source: `/internal/classification/db/recipes/${file}`,
+        source: `/pkg/classification/db/recipes/${file}`,
       }
     }),
   )
   return result
 }
 module.exports = async function () {
-  const recipes = await fetchData("../internal/classification/db/recipes/")
+  const recipes = await fetchData("../pkg/classification/db/recipes/")
   return recipes
 }

docs/_data/rules.js

@@ -9,7 +9,7 @@ const source = "bearer/bearer-rules"
 const rulesPath = "_tmp/rules-data"
 const excludeDirectories = [".github", "scripts"]
 const gitleaksInternalRule =
-  "../internal/commands/process/settings/built_in_rules/third_party/gitleaks/secret_detection.yml"
+  "../pkg/commands/process/settings/rules/built_in/third_party/gitleaks/secret_detection.yml"
 
 const counts = {
   languages: {},

docs/contributing/code.md

@@ -65,13 +65,13 @@ go test ./...
 Running classification tests:
 
 ```bash
-go test ./internal/classification/... -count=1
+go test ./pkg/classification/... -count=1
 ```
 
 Running a single specific test:
 
 ```bash
-go test -run ^TestSchema$ ./internal/classification/schema -count=1
+go test -run ^TestSchema$ ./pkg/classification/schema -count=1
 ```
 
 ### Integration testing

docs/contributing/recipes.md

@@ -6,11 +6,11 @@ title: Add or update a recipe
 
 Recipes are part of how Bearer CLI makes connections between your code and other sources. These are things like data stores, APIs, and internal services. They work by providing information about endpoints, API base urls, package information, etc.
 
-Recipes are located at `bearer/internal/classification/db/recipes/`.
+Recipes are located at `bearer/pkg/classification/db/recipes/`.
 
 ```
 .
-├ internal/
+├ pkg/
 │  └ classification/
 │       └  db/
 │           └  recipes/
@@ -42,7 +42,7 @@ Each recipe consists of a `JSON` file containing the following properties:
   - `internal_service` subtypes:
     - `message_bus`
 
-If any of the existing properties and available values don't meet the needs of your new recipe, [open a new issue]({{meta.sourcePath}}/issues/new/choose). You can view the existing recipes [in the GitHub repo]({{meta.sourcePath}}/tree/main/internal/classification/db/recipes).
+If any of the existing properties and available values don't meet the needs of your new recipe, [open a new issue]({{meta.sourcePath}}/issues/new/choose). You can view the existing recipes [in the GitHub repo]({{meta.sourcePath}}/tree/main/pkg/classification/db/recipes).
 
 ## Generating a UUID
 

docs/explanations/reports.md

@@ -110,7 +110,7 @@ By default, Bearer CLI maps all subjects to “User”, but you can override thi
 bearer scan . --report=privacy --data-subject-mapping=/path/to/mappings.json
 ```
 
-The custom map file should follow the format used by [subject_mapping.json]({{meta.sourcePath}}/blob/main/internal/classification/db/subject_mapping.json). Replace a key’s value with the higher-level subject you’d like to associate it with. Some examples might include Customer, Employee, Client, Patient, etc. Bearer CLI will use your replacement file instead of the default, so make sure to include any and all subjects you want reported.
+The custom map file should follow the format used by [subject_mapping.json]({{meta.sourcePath}}/blob/main/pkg/classification/db/subject_mapping.json). Replace a key’s value with the higher-level subject you’d like to associate it with. Some examples might include Customer, Employee, Client, Patient, etc. Bearer CLI will use your replacement file instead of the default, so make sure to include any and all subjects you want reported.
 
 ## Data Flow Report
 

docs/explanations/scanners.md

@@ -50,6 +50,6 @@ Detected: Password in URL
 File: ../../OWASP/NodeGoat/README.md:59
 ```
 
-You can see a full list of [built-in patterns](https://github.com/Bearer/bearer/blob/main/internal/detectors/gitleaks/gitlab_config.toml).
+You can see a full list of [built-in patterns](https://github.com/Bearer/bearer/blob/main/pkg/detectors/gitleaks/gitlab_config.toml).
 
 ⚠️ Secret detection patterns are not configurable today. If this is something you'd like to see, please open an [issue](https://github.com/Bearer/bearer/issues).

docs/guides/custom-rule.md

@@ -34,7 +34,7 @@ To better understand the structure of a rule file, let’s look at each key:
   - `id`: A unique identifier. Internal rules are named `lang_framework_rule_name`. For rules targeting the language core, `lang` is used instead of a framework name. For example `ruby_lang_logger` and `ruby_rails_logger`. For custom rules, you may consider appending your org name.
   - `description`: A brief, one-sentence description of the rule. The best practice is to make this an actionable “rule” phrase, such as “Do X” or “Do not do X in Y”.
   - `cwe_id`: The associated list of [CWE](https://cwe.mitre.org/) identifiers. (Optional)
-  - `associated_recipe`: Links the rule to a [recipe]({{meta.sourcePath}}/tree/main/internal/classification/db/recipes). Useful for associating a rule with a third party. Example: “Sentry” (Optional)
+  - `associated_recipe`: Links the rule to a [recipe]({{meta.sourcePath}}/tree/main/pkg/classification/db/recipes). Useful for associating a rule with a third party. Example: “Sentry” (Optional)
   - `remediation_message`: Used for internal rules, this builds the documentation page for a rule. (Optional)
   - `documentation_url`: Used to pass custom documentation URL for the security report. This can be useful for linking to your own internal documentation or policies. By default, all rules in the main repo will automatically generate a link to the rule on [docs.bearer.com](/). (Optional)
 - `auxiliary`: Allows you to define helper rules and detectors to make pattern-building more robust. Auxiliary rules contain a unique `id` and their own `patterns` in the same way rules do. You’re unlikely to use this regularly. See the [weak_encryption](https://github.com/Bearer/bearer-rules/blob/main/ruby/lang/weak_encryption.yml) rule for examples. In addition, see our advice on how to avoid [variable joining](#variable-joining) in auxiliary rules. (Optional)

docs/guides/privacy.md

@@ -106,7 +106,7 @@ This will allow team members to import the report into spreadsheets or their pre
 
 ## Subject mapping
 
-Bearer CLI uses "User" as the default data subject. To override this, you can copy the [subject_mapping.json](https://github.com/bearer/bearer/blob/main/internal/classification/db/subject_mapping.json) and customize it to your needs. Then, use the `--data-subject-mapping` flag to use your mappings instead. This will use your supplied mapping file instead of the default.
+Bearer CLI uses "User" as the default data subject. To override this, you can copy the [subject_mapping.json](https://github.com/bearer/bearer/blob/main/pkg/classification/db/subject_mapping.json) and customize it to your needs. Then, use the `--data-subject-mapping` flag to use your mappings instead. This will use your supplied mapping file instead of the default.
 
 ```bash
 bearer scan . --report privacy --data-subject-mapping /path/to/mappings.json

docs/reference/commands.njk

@@ -3,7 +3,7 @@ title: Commands
 layout: layouts/doc.njk
 ---
 {# Welcome :wave:. The content of this page is automatically generated based on Bearer's CLI help files.
-They can be found here: https://github.com/Bearer/bearer/tree/main/internal/commands
+They can be found here: https://github.com/Bearer/bearer/tree/main/pkg/commands
  #}
 
 {% set items = [bearer_scan, bearer_init, bearer_ignore_add, bearer_ignore_show, bearer_ignore_remove, bearer_ignore_pull, bearer_ignore_migrate, bearer_version] %}

docs/reference/datatypes.njk

@@ -3,7 +3,7 @@ title: Data Types
 layout: layouts/doc.njk
 ---
 {# This content of this page is generated. To edit individual types or category text,
-edit the files located at https://github.com/Bearer/bearer/tree/main/internal/classification/db #}
+edit the files located at https://github.com/Bearer/bearer/tree/main/pkg/classification/db #}
 {% renderTemplate "liquid,md",
 datatypes %}
 # Supported Data Types

e2e/flags/report_flags_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	"github.com/bearer/bearer/e2e/internal/testhelper"
-	"github.com/bearer/bearer/internal/util/tmpfile"
+	"github.com/bearer/bearer/pkg/util/tmpfile"
 	"github.com/bradleyjkemp/cupaloy"
 )
 

external/run/run.go

@@ -0,0 +1,97 @@
+package run
+
+import (
+	"os"
+	"regexp"
+
+	"github.com/bearer/bearer/pkg/classification/schema"
+	"github.com/bearer/bearer/pkg/commands"
+	"github.com/bearer/bearer/pkg/engine"
+	"github.com/bearer/bearer/pkg/engine/implementation"
+	"github.com/bearer/bearer/pkg/languages"
+	"github.com/bearer/bearer/pkg/scanner/ast/query"
+	"github.com/bearer/bearer/pkg/scanner/ast/tree"
+	"github.com/bearer/bearer/pkg/scanner/detectors/common"
+	"github.com/bearer/bearer/pkg/scanner/detectors/types"
+	"github.com/bearer/bearer/pkg/scanner/language"
+	"github.com/bearer/bearer/pkg/scanner/ruleset"
+	"github.com/bearer/bearer/pkg/util/regex"
+	"github.com/bearer/bearer/pkg/util/stringutil"
+)
+
+type Object = common.Object
+type Property = common.Property
+type String = common.String
+
+type Engine = engine.Engine
+
+type Analyzer = language.Analyzer
+type Language = language.Language
+type Pattern = language.Pattern
+type PatternBase = language.PatternBase
+type PatternVariable = language.PatternVariable
+type Scope = language.Scope
+
+type Query = query.Query
+type Set = query.Set
+
+type Rule = ruleset.Rule
+
+type Classifier = schema.Classifier
+
+type Builder = tree.Builder
+type Node = tree.Node
+
+type Context = types.Context
+type Detection = types.Detection
+type Detector = types.Detector
+type DetectorBase = types.DetectorBase
+
+var BuiltinObjectRule = ruleset.BuiltinObjectRule
+var BuiltinStringRule = ruleset.BuiltinStringRule
+
+func GetNonVirtualObjects(detectorContext types.Context, node *tree.Node) ([]*types.Detection, error) {
+	return common.GetNonVirtualObjects(detectorContext, node)
+}
+
+func StripQuotes(input string) string {
+	return stringutil.StripQuotes(input)
+}
+
+func ConcatenateChildStrings(node *tree.Node, detectorContext types.Context) ([]interface{}, error) {
+	return common.ConcatenateChildStrings(node, detectorContext)
+}
+
+func ConcatenateAssignEquals(node *tree.Node, detectorContext types.Context) ([]interface{}, error) {
+	return common.ConcatenateAssignEquals(node, detectorContext)
+}
+
+func ProjectObject(node *tree.Node, detectorContext types.Context, objectNode *tree.Node, objectName string, propertyName string, isPropertyAccess bool) ([]interface{}, error) {
+	return common.ProjectObject(node, detectorContext, objectNode, objectName, propertyName, isPropertyAccess)
+}
+
+func ReplaceAllWithSubmatches(pattern *regexp.Regexp, input string, replace func(submatches []string) (string, error)) (string, error) {
+	return regex.ReplaceAllWithSubmatches(pattern, input, replace)
+}
+
+func NewScope(parent *language.Scope) *language.Scope {
+	return language.NewScope(parent)
+}
+
+func NewEngine(languages []Language) Engine {
+	return implementation.New(languages)
+}
+
+func DefaultLanguages() []Language {
+	return languages.Default()
+}
+
+func Run(version, commitSHA string, engine Engine) {
+	err := commands.NewApp(version, commitSHA, engine).Execute()
+	engine.Close()
+
+	if err != nil {
+		// error messages are printed by the framework
+		os.Exit(1)
+	}
+}