From 9a613660f6a5bf616accfa425eded5ca598a067c Mon Sep 17 00:00:00 2001 From: David Roe Date: Tue, 24 Oct 2023 16:49:13 +0100 Subject: [PATCH] ci: add version comparison workflow (#1350) --- .github/workflows/kpi_scans.yml | 3 +- .github/workflows/kpi_scans_staging.yml | 3 +- .github/workflows/version_comparison.yml | 116 +++++++++++++++++++++++ kpi_scan/kpi_repo_list.json | 18 ---- kpi_scan/kpi_repo_list.json5 | 36 +++++++ 5 files changed, 154 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/version_comparison.yml delete mode 100644 kpi_scan/kpi_repo_list.json create mode 100644 kpi_scan/kpi_repo_list.json5 diff --git a/.github/workflows/kpi_scans.yml b/.github/workflows/kpi_scans.yml index 772ca5129..3f7467bde 100644 --- a/.github/workflows/kpi_scans.yml +++ b/.github/workflows/kpi_scans.yml @@ -27,8 +27,7 @@ jobs: - uses: actions/checkout@v4 - id: load_json run : | - content=$(cat ./kpi_scan/kpi_repo_list.json | jq -c) - echo "matrix=$content" >> $GITHUB_OUTPUT + echo "matrix=$(npx --yes json5 ./kpi_scan/kpi_repo_list.json5)" >> $GITHUB_OUTPUT build: needs: [build_and_push_docker_image, load_repo_list] name: Run KPI scans diff --git a/.github/workflows/kpi_scans_staging.yml b/.github/workflows/kpi_scans_staging.yml index fb0c6cb61..807d2eded 100644 --- a/.github/workflows/kpi_scans_staging.yml +++ b/.github/workflows/kpi_scans_staging.yml @@ -27,8 +27,7 @@ jobs: - uses: actions/checkout@v4 - id: load_json run : | - content=$(cat ./kpi_scan/kpi_repo_list.json | jq -c) - echo "matrix=$content" >> $GITHUB_OUTPUT + echo "matrix=$(npx --yes json5 ./kpi_scan/kpi_repo_list.json5)" >> $GITHUB_OUTPUT build: needs: [build_and_push_docker_image, load_repo_list] name: Run Staging KPI scans diff --git a/.github/workflows/version_comparison.yml b/.github/workflows/version_comparison.yml new file mode 100644 index 000000000..9027ab4a3 --- /dev/null +++ b/.github/workflows/version_comparison.yml @@ -0,0 +1,116 @@ +name: Version Comparison +on: + workflow_dispatch: + inputs: + baseRef: + description: 'Base CLI ref (tag/branch/SHA)' + baseRulesRef: + description: 'Base rules ref' + testRef: + description: 'Test CLI ref (tag/branch/SHA)' + testRulesRef: + description: 'Test rules ref' + +jobs: + setup: + name: Setup version comparison + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load_repo_list.outputs.matrix }} + cache_key: ${{ steps.cache_key.outputs.value }} + steps: + - uses: actions/checkout@v4 + - id: load_repo_list + name: Load KPI repository list + run : | + echo "matrix=$(npx --yes json5 ./kpi_scan/kpi_repo_list.json5)" >> $GITHUB_OUTPUT + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.21 + - id: cache_key + name: Create cache key + run: | + echo "value=cache-${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_OUTPUT + - name: Create cache folder + run: mkdir bearer-comparison + - name: Checkout base CLI + uses: actions/checkout@v4 + with: + repository: bearer/bearer + ref: ${{ inputs.baseRef }} + path: base-cli + - name: Checkout base rules + uses: actions/checkout@v4 + with: + repository: bearer/bearer-rules + ref: ${{ inputs.baseRulesRef }} + path: bearer-comparison/base-rules + - name: Build base CLI + run: | + cd ./base-cli + go build -o ../bearer-comparison/base-bearer ./cmd/bearer/main.go + - name: Checkout test CLI + uses: actions/checkout@v4 + with: + repository: bearer/bearer + ref: ${{ inputs.testRef }} + path: test-cli + - name: Checkout test rules + uses: actions/checkout@v4 + with: + repository: bearer/bearer-rules + ref: ${{ inputs.testRulesRef }} + path: bearer-comparison/test-rules + - name: Build test CLI + run: | + cd ./test-cli + go build -o ../bearer-comparison/test-bearer ./cmd/bearer/main.go + - name: Cache CLIs and rules + uses: actions/cache/save@v3 + with: + path: bearer-comparison + key: ${{ steps.cache_key.outputs.value }} + test: + needs: [setup] + name: Scan ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + matrix: ${{fromJson(needs.setup.outputs.matrix)}} + fail-fast: false + steps: + - name: Restore CLIs and rules + uses: actions/cache/restore@v3 + with: + path: bearer-comparison + key: ${{ needs.setup.outputs.cache_key }} + - name: Checkout KPI repo + run: | + git clone --single-branch --depth 1 --no-tags ${{ matrix.repository_url }} ${{ matrix.name }} + - name: Run base scan + run: | + ./bearer-comparison/base-bearer scan ${{ matrix.name }} \ + --format json \ + --exit-code 0 \ + --disable-default-rules \ + --external-rule-dir ./bearer-comparison/base-rules/rules \ + --force \ + --disable-version-check \ + --quiet \ + --hide-progress-bar \ + | jq > base.json + - name: Run test scan + run: | + ./bearer-comparison/test-bearer scan ${{ matrix.name }} \ + --format json \ + --exit-code 0 \ + --disable-default-rules \ + --external-rule-dir ./bearer-comparison/test-rules/rules \ + --force \ + --disable-version-check \ + --quiet \ + --hide-progress-bar \ + | jq > test.json + - run: | + diff -u base.json test.json + diff --git a/kpi_scan/kpi_repo_list.json b/kpi_scan/kpi_repo_list.json deleted file mode 100644 index 422b7d9aa..000000000 --- a/kpi_scan/kpi_repo_list.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "repository_url": [ - "https://github.com/juice-shop/juice-shop", - "https://github.com/OWASP/railsgoat", - "https://github.com/OWASP/NodeGoat", - "https://github.com/WebGoat/WebGoat", - "https://github.com/forem/forem", - "https://github.com/freeCodeCamp/chapter", - "https://github.com/mastodon/mastodon", - "https://github.com/frab/frab", - "https://github.com/discourse/discourse", - "https://github.com/diaspora/diaspora", - "https://github.com/TryGhost/Ghost", - "https://github.com/wekan/wekan", - "https://gitlab.com/gitlab-org/gitlab", - "https://github.com/grafana/grafana" - ] -} \ No newline at end of file diff --git a/kpi_scan/kpi_repo_list.json5 b/kpi_scan/kpi_repo_list.json5 new file mode 100644 index 000000000..89676c786 --- /dev/null +++ b/kpi_scan/kpi_repo_list.json5 @@ -0,0 +1,36 @@ +{ + "include": [ + // ruby + { "name": "railsgoat", "repository_url": "https://github.com/Bearer/railsgoat" }, + { "name": "mastodon", "repository_url": "https://github.com/mastodon/mastodon" }, + { "name": "frab", "repository_url": "https://github.com/frab/frab" }, + { "name": "discourse", "repository_url": "https://github.com/discourse/discourse" }, + { "name": "diaspora", "repository_url": "https://github.com/diaspora/diaspora" }, + { "name": "gitlab", "repository_url": "https://gitlab.com/gitlab-org/gitlab" }, + { "name": "chatwoot", "repository_url": "https://github.com/chatwoot/chatwoot" }, + { "name": "postal", "repository_url": "https://github.com/postalserver/postal" }, + { "name": "forem", "repository_url": "https://github.com/forem/forem" }, + { "name": "openstreetmap-website", "repository_url": "https://github.com/openstreetmap/openstreetmap-website" }, + { "name": "loomio", "repository_url": "https://github.com/loomio/loomio" }, + { "name": "rdv-solidarites.fr", "repository_url": "https://github.com/betagouv/rdv-solidarites.fr" }, + // javascript + { "name": "juice-shop", "repository_url": "https://github.com/Bearer/juice-shop" }, + { "name": "NodeGoat", "repository_url": "https://github.com/Bearer/NodeGoat" }, + { "name": "chapter", "repository_url": "https://github.com/freeCodeCamp/chapter" }, + { "name": "Ghost", "repository_url": "https://github.com/TryGhost/Ghost" }, + { "name": "wekan", "repository_url": "https://github.com/wekan/wekan" }, + { "name": "backstage", "repository_url": "https://github.com/backstage/backstage" }, + { "name": "medusa", "repository_url": "https://github.com/medusajs/medusa" }, + { "name": "ToolJet", "repository_url": "https://github.com/ToolJet/ToolJet" }, + { "name": "grafana", "repository_url": "https://github.com/grafana/grafana" }, + { "name": "mattermost-server", "repository_url": "https://github.com/mattermost/mattermost-server" }, + { "name": "Rocket.Chat", "repository_url": "https://github.com/RocketChat/Rocket.Chat" }, + // java + { "name": "WebGoat", "repository_url": "https://github.com/Bearer/WebGoat" }, + { "name": "BenchmarkJava", "repository_url": "https://github.com/OWASP-Benchmark/BenchmarkJava" }, + // php + { "name": "OWASPWebGoatPHP", "repository_url": "https://github.com/OWASP/OWASPWebGoatPHP" }, + { "name": "Vulnerable-Web-Application", "repository_url": "https://github.com/OWASP/Vulnerable-Web-Application" }, + { "name": "mediawiki", "repository_url": "https://github.com/wikimedia/mediawiki" } + ] +}