chore: add new evaluator

scripts/invoke.sh

@@ -8,6 +8,7 @@ test_location=$2
 rule_id=$3
 rule_loc=$PWD/rules
 BEARER_VERSION=${BEARER_VERSION=latest}
+FORMAT=${FORMAT=json}
 
 filename=$(basename $test_location)
 tmp_location=/tmp/bearer-scan/$filename
@@ -21,7 +22,7 @@ if [ -n "$BEARER_WORKSPACE" ]; then
     --quiet \
     --disable-default-rules=true \
     --external-rule-dir=$rule_loc \
-    --format=json \
+    --format=${FORMAT} \
     --disable-version-check \
     --force \
     --exit-code=0 \
@@ -37,7 +38,7 @@ else
       --only-rule=$rule_id \
       --disable-default-rules=true \
       --external-rule-dir=/tmp/rules \
-      --format=json \
+      --format=${FORMAT} \
       --quiet \
       --disable-version-check \
       --exit-code=0 \

tests/helper.js

@@ -29,7 +29,7 @@ exports.createInvoker = (ruleId, ruleFile, testBase) => {
 
     results = JSON.parse(out)
     let findings = []
-    for (const [key, values] of Object.entries(results)) {
+    for (const [_severity, values] of Object.entries(results)) {
       for (const [value] in values) {
         findings.push({
           // severity: key,
@@ -59,3 +59,37 @@ bearer scan ${testBase} --only-rule ${ruleId} --log-level trace`
     return JSON.stringify(results, null, 2)
   }
 }
+
+function difference(setA, setB) {
+  const diff = new Set(setA)
+
+  for (const elem of setB) {
+    diff.delete(elem)
+  }
+
+  return Array.from(diff)
+}
+
+exports.createNewInvoker = (ruleId, ruleFile, testBase) => {
+  return (testCase) => {
+    const out = execSync(
+      `FORMAT=jsonv2 ./scripts/invoke.sh ${ruleFile} ${testBase}${testCase} ${ruleId}`
+    ).toString()
+
+    results = JSON.parse(out)
+    let findings = []
+    for (const result of results.findings) {
+      findings.push(`${result.id}:${result.source.start}`)
+    }
+
+    let expectedFindings = []
+    for (const result of results.expected_findings) {
+      expectedFindings.push(`${result.rule_id}:${result.location.start}`)
+    }
+
+    return {
+      Extra: difference(new Set(findings), new Set(expectedFindings)),
+      Missing: difference(new Set(expectedFindings), new Set(findings)),
+    }
+  }
+}

tests/javascript/lang/eval_user_input/test.js

@@ -1,37 +1,43 @@
-const { createInvoker, getEnvironment } = require("../../../helper.js")
+const {
+  createInvoker,
+  createNewInvoker,
+  getEnvironment,
+} = require("../../../helper.js")
 const { ruleId, ruleFile, testBase } = getEnvironment(__dirname)
 
 describe(ruleId, () => {
   const invoke = createInvoker(ruleId, ruleFile, testBase)
-
+  const newInvoke = createNewInvoker(ruleId, ruleFile, testBase)
 
   test("eval", () => {
     const testCase = "eval.js"
-    expect(invoke(testCase)).toMatchSnapshot();
+    expect(invoke(testCase)).toMatchSnapshot()
+  })
+
+  test("new-eval", () => {
+    const testCase = "eval.js"
+    const results = newInvoke(testCase)
+    expect(results.Missing).toEqual([])
+    expect(results.Extra).toEqual([])
   })
-
 
   test("new_function", () => {
     const testCase = "new_function.js"
-    expect(invoke(testCase)).toMatchSnapshot();
+    expect(invoke(testCase)).toMatchSnapshot()
   })
-
 
   test("secure", () => {
     const testCase = "secure.js"
-    expect(invoke(testCase)).toMatchSnapshot();
+    expect(invoke(testCase)).toMatchSnapshot()
   })
-
 
   test("set_interval", () => {
     const testCase = "set_interval.js"
-    expect(invoke(testCase)).toMatchSnapshot();
+    expect(invoke(testCase)).toMatchSnapshot()
   })
-
 
   test("set_timeout", () => {
     const testCase = "set_timeout.js"
-    expect(invoke(testCase)).toMatchSnapshot();
+    expect(invoke(testCase)).toMatchSnapshot()
   })
-
-})
+})

tests/javascript/lang/eval_user_input/testdata/eval.js

@@ -8,12 +8,14 @@ app.use(helmet.hidePoweredBy())
 app.post("/:id", (req, res) => {
   userInput = req.params.id
   var command = "new Function('" + userInput + "')"
+  // bearer:expected javascript_lang_eval_user_input
   return eval(command)
 })
 
-const vm = require('node:vm');
+const vm = require("node:vm")
 
 exports.handler = async function (event, _context) {
-  const context = event["params"]["context"];
-  eval(context);
+  const context = event["params"]["context"]
+  // bearer:expected javascript_lang_eval_user_input
+  eval(context)
 }