ci: add version comparison workflow

.github/workflows/kpi_scans.yml

@@ -27,8 +27,7 @@ jobs:
       - uses: actions/checkout@v4
       - id: load_json
         run : |
-          content=$(cat ./kpi_scan/kpi_repo_list.json | jq -c)
-          echo "matrix=$content" >> $GITHUB_OUTPUT
+          echo "matrix=$(npx --yes json5 ./kpi_scan/kpi_repo_list.json5)" >> $GITHUB_OUTPUT
   build:
     needs: [build_and_push_docker_image, load_repo_list]
     name: Run KPI scans

.github/workflows/kpi_scans_staging.yml

@@ -27,8 +27,7 @@ jobs:
       - uses: actions/checkout@v4
       - id: load_json
         run : |
-          content=$(cat ./kpi_scan/kpi_repo_list.json | jq -c)
-          echo "matrix=$content" >> $GITHUB_OUTPUT
+          echo "matrix=$(npx --yes json5 ./kpi_scan/kpi_repo_list.json5)" >> $GITHUB_OUTPUT
   build:
     needs: [build_and_push_docker_image, load_repo_list]
     name: Run Staging KPI scans

.github/workflows/version_comparison.yml

@@ -0,0 +1,116 @@
+name: Version Comparison
+on:
+  workflow_dispatch:
+    inputs:
+      baseRef:
+        description: 'Base CLI ref (tag/branch/SHA)'
+      baseRulesRef:
+        description: 'Base rules ref'
+      testRef:
+        description: 'Test CLI ref (tag/branch/SHA)'
+      testRulesRef:
+        description: 'Test rules ref'
+
+jobs:
+  setup:
+    name: Setup version comparison
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.load_repo_list.outputs.matrix }}
+      cache_key: ${{ steps.cache_key.outputs.value }}
+    steps:
+      - uses: actions/checkout@v4
+      - id: load_repo_list
+        name: Load KPI repository list
+        run : |
+          echo "matrix=$(npx --yes json5 ./kpi_scan/kpi_repo_list.json5)" >> $GITHUB_OUTPUT
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21
+      - id: cache_key
+        name: Create cache key
+        run: |
+          echo "value=cache-${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_OUTPUT
+      - name: Create cache folder
+        run: mkdir bearer-comparison
+      - name: Checkout base CLI
+        uses: actions/checkout@v4
+        with:
+          repository: bearer/bearer
+          ref: ${{ inputs.baseRef }}
+          path: base-cli
+      - name: Checkout base rules
+        uses: actions/checkout@v4
+        with:
+          repository: bearer/bearer-rules
+          ref: ${{ inputs.baseRulesRef }}
+          path: bearer-comparison/base-rules
+      - name: Build base CLI
+        run: |
+          cd ./base-cli
+          go build -o ../bearer-comparison/base-bearer ./cmd/bearer/main.go
+      - name: Checkout test CLI
+        uses: actions/checkout@v4
+        with:
+          repository: bearer/bearer
+          ref: ${{ inputs.testRef }}
+          path: test-cli
+      - name: Checkout test rules
+        uses: actions/checkout@v4
+        with:
+          repository: bearer/bearer-rules
+          ref: ${{ inputs.testRulesRef }}
+          path: bearer-comparison/test-rules
+      - name: Build test CLI
+        run: |
+          cd ./test-cli
+          go build -o ../bearer-comparison/test-bearer ./cmd/bearer/main.go
+      - name: Cache CLIs and rules
+        uses: actions/cache/save@v3
+        with:
+          path: bearer-comparison
+          key: ${{ steps.cache_key.outputs.value }}
+  test:
+    needs: [setup]
+    name: Scan ${{ matrix.name }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{fromJson(needs.setup.outputs.matrix)}}
+      fail-fast: false
+    steps:
+      - name: Restore CLIs and rules
+        uses: actions/cache/restore@v3
+        with:
+          path: bearer-comparison
+          key: ${{ needs.setup.outputs.cache_key }}
+      - name: Checkout KPI repo
+        run: |
+          git clone --single-branch --depth 1 --no-tags ${{ matrix.repository_url }} ${{ matrix.name }}
+      - name: Run base scan
+        run: |
+          ./bearer-comparison/base-bearer scan ${{ matrix.name }} \
+            --format json \
+            --exit-code 0 \
+            --disable-default-rules \
+            --external-rule-dir ./bearer-comparison/base-rules/rules \
+            --force \
+            --disable-version-check \
+            --quiet \
+            --hide-progress-bar \
+            | jq > base.json
+      - name: Run test scan
+        run: |
+          ./bearer-comparison/test-bearer scan ${{ matrix.name }} \
+            --format json \
+            --exit-code 0 \
+            --disable-default-rules \
+            --external-rule-dir ./bearer-comparison/test-rules/rules \
+            --force \
+            --disable-version-check \
+            --quiet \
+            --hide-progress-bar \
+            | jq > test.json
+      - run: |
+          diff -u base.json test.json
+

kpi_scan/kpi_repo_list.json5

@@ -0,0 +1,36 @@
+{
+  "include": [
+    // ruby
+    { "name": "railsgoat", "repository_url": "https://github.com/Bearer/railsgoat" },
+    { "name": "mastodon", "repository_url": "https://github.com/mastodon/mastodon" },
+    { "name": "frab", "repository_url": "https://github.com/frab/frab" },
+    { "name": "discourse", "repository_url": "https://github.com/discourse/discourse" },
+    { "name": "diaspora", "repository_url": "https://github.com/diaspora/diaspora" },
+    { "name": "gitlab", "repository_url": "https://gitlab.com/gitlab-org/gitlab" },
+    { "name": "chatwoot", "repository_url": "https://github.com/chatwoot/chatwoot" },
+    { "name": "postal", "repository_url": "https://github.com/postalserver/postal" },
+    { "name": "forem", "repository_url": "https://github.com/forem/forem" },
+    { "name": "openstreetmap-website", "repository_url": "https://github.com/openstreetmap/openstreetmap-website" },
+    { "name": "loomio", "repository_url": "https://github.com/loomio/loomio" },
+    { "name": "rdv-solidarites.fr", "repository_url": "https://github.com/betagouv/rdv-solidarites.fr" },
+    // javascript
+    { "name": "juice-shop", "repository_url": "https://github.com/Bearer/juice-shop" },
+    { "name": "NodeGoat", "repository_url": "https://github.com/Bearer/NodeGoat" },
+    { "name": "chapter", "repository_url": "https://github.com/freeCodeCamp/chapter" },
+    { "name": "Ghost", "repository_url": "https://github.com/TryGhost/Ghost" },
+    { "name": "wekan", "repository_url": "https://github.com/wekan/wekan" },
+    { "name": "backstage", "repository_url": "https://github.com/backstage/backstage" },
+    { "name": "medusa", "repository_url": "https://github.com/medusajs/medusa" },
+    { "name": "ToolJet", "repository_url": "https://github.com/ToolJet/ToolJet" },
+    { "name": "grafana", "repository_url": "https://github.com/grafana/grafana" },
+    { "name": "mattermost-server", "repository_url": "https://github.com/mattermost/mattermost-server" },
+    { "name": "Rocket.Chat", "repository_url": "https://github.com/RocketChat/Rocket.Chat" },
+    // java
+    { "name": "WebGoat", "repository_url": "https://github.com/Bearer/WebGoat" },
+    { "name": "BenchmarkJava", "repository_url": "https://github.com/OWASP-Benchmark/BenchmarkJava" },
+    // php
+    { "name": "OWASPWebGoatPHP", "repository_url": "https://github.com/OWASP/OWASPWebGoatPHP" },
+    { "name": "Vulnerable-Web-Application", "repository_url": "https://github.com/OWASP/Vulnerable-Web-Application" },
+    { "name": "mediawiki", "repository_url": "https://github.com/wikimedia/mediawiki" }
+  ]
+}