rule-validate.yml

name: Rules PR CI

on:
  push:
    branches: [ "main", "test-rules" ]
  pull_request_target:
    branches: [ "**" ]
  workflow_dispatch: {}
  issue_comment:
    types: [ created ]
  merge_group: {}

concurrency:
  # For pull_request_target workflows we want to use head_ref -- the branch triggering the workflow. Otherwise,
  # use ref, which is the branch for a push event or workflow trigger. And for an issue comment just give up grouping.
  group: ${{ github.event_name == 'pull_request_target' && github.head_ref || (github.event_name == 'issue_comment' && github.run_id || github.ref) }}
  cancel-in-progress: ${{ github.event_name == 'pull_request_target' }}

jobs:
  tests:
    name: Run Rule Validation
    runs-on: ubuntu-20.04
    permissions:
      contents: write
      issues: read
      pull-requests: read
      checks: write
    if: github.event_name != 'issue_comment' || github.event.issue.pull_request && contains(github.event.comment.body, '/mql-mimic-exempt')

    steps:
      - name: Set up yq
        uses: mikefarah/yq@v4.27.3

      - name: Get PR branch
        if: github.event_name == 'issue_comment'
        uses: alessbell/pull-request-comment-branch@v1.1 # Fork of xt0rted/pull-request-comment-branch, see https://github.com/xt0rted/pull-request-comment-branch/issues/322
        id: comment_branch

      - name: Get Refs
        id: get_head_ref
        run: |
          # Accurate for push events, merge queues, and workflow dispatch.
          head_ref="${{ github.ref }}"
          repo="${{ github.repository }}"

          if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
            head_ref="${{ github.head_ref }}"
            repo="${{ github.event.pull_request.head.repo.full_name }}"
          elif [[ "${{ github.event_name }}" == 'issue_comment' ]]; then
            # Rely on comment_branch to figure out the head and base
            head_ref="${{ steps.comment_branch.outputs.head_ref }}"
            repo="${{ steps.comment_branch.outputs.head_owner }}/${{ steps.comment_branch.outputs.head_repo }}"
          fi
          
          echo "##[set-output name=head_ref;]$head_ref"
          echo "##[set-output name=repo;]$repo"

      - name: Checkout
        uses: actions/checkout@v3
        with:
          repository: ${{ steps.get_head_ref.outputs.repo }}
          ref: ${{ steps.get_head_ref.outputs.head_ref }}
          fetch-depth: 0

      - name: Get Refs
        id: get_base_ref
        run: |
          run_all=""
          base_ref=""

          if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
            # Detect changes based on whatever we're merging into.
            base_ref="${{ github.base_ref }}"
          elif [[ "${{ github.event_name }}" == 'push' || "${{ github.event_name }}" == 'merge_group' ]]; then
            # Detect changes based on the previous commit
            base_ref="$(git rev-parse HEAD^)"
          elif [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then
            # Run on a target, so run for all rules.
            run_all="true"
          elif [[ "${{ github.event_name }}" == 'issue_comment' ]]; then
            # Rely on comment_branch to figure out  base
            base_ref="${{ steps.comment_branch.outputs.base_ref }}"
          fi
          
          echo "##[set-output name=run_all;]$run_all"
          echo "##[set-output name=base_ref;]$base_ref"

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Add Rule IDs as Needed & Check for Duplicates
        if: github.event_name != 'issue_comment'
        # Run before testing, just in case this could invalidate the rule itself
        run: |
          pip install -r scripts/generate-rule-ids/requirements.txt
          python scripts/generate-rule-ids/main.py

      - name: Validate Rules
        if: github.event_name != 'issue_comment'
        run: |
          echo '{"rules_or_queries": [' > bulk_validate_request.json
          
          file_count=$(ls -1 {*-rules/*.yml,insights/**/*.yml} | wc -l)
          counter=0
  
          for f in *-rules/*.yml
          do
            counter=$((counter + 1))
            yq -o=json eval 'del(.type)' "$f" >> bulk_validate_request.json
            if [[ $counter -ne $file_count ]]; then
              echo "," >> bulk_validate_request.json
            fi
          done

          for f in insights/**/*.yml
          do
            counter=$((counter + 1))
            yq -o=json eval 'del(.type) | .source = "length([\n\n" + .source + "\n]) >= 0"' "$f" >> bulk_validate_request.json
          
            if [[ $counter -ne $file_count ]]; then
              echo "," >> bulk_validate_request.json
            fi
          done

          echo "]}" >> bulk_validate_request.json
          http_code=$(curl -H "Content-Type: application/json" -X POST -d @bulk_validate_request.json -o response.txt -w "%{http_code}" --silent https://play.sublime.security/v1/rules/bulk_validate)
          echo '' >> response.txt
          cat response.txt
          if [[ "$http_code" != "200" ]]; then
            echo "Unexpected response $http_code"
            exit 1
          fi

      - name: Verify no .yaml files exist
        if: github.event_name != 'issue_comment'
        run: |
          ! /bin/sh -c 'ls **/*.yaml'

          
      - name: Verify no .yml files exist in the top directory
        if: github.event_name != 'issue_comment'
        run: |
          ! /bin/sh -c 'ls *.yml'

      - name: Commit & Push Results, if needed
        if: github.event_name != 'issue_comment'
        id: final_basic_validation
        run: |
          rm response.txt
          rm bulk_validate_request.json

          if [ -z "$(git status --porcelain)" ]; then
            echo "No files changed, nothing to do"
            exit 0
          fi

          git config user.name 'ID Generator'
          git config user.email 'hello@sublimesecurity.com'
          git add **/*.yml
          git commit -m "Auto add rule ID"
          # This will only work when running for a pull_request_target, but rather than filter we'll let this expose
          # any issues.
          git push origin ${{ steps.get_head_ref.outputs.head_ref }}

      - name: Get the head SHA
        id: get_head
        if: ${{ always() }}
        run: echo "##[set-output name=HEAD;]$(git rev-parse HEAD)"

      # When we add a commit, GitHub won't trigger actions on the auto commit, so we're missing a required check on the
      # HEAD commit.
      # Various alternatives were explored, but all run into issues when dealing with forks. This sets a "Check" for
      # the latest commit, and we can depend on that as a required check.
      - name: "Create a check run"
        uses: actions/github-script@v6
        if: (github.event_name == 'pull_request_target' || github.event_name == 'merge_group' ) && always()
        env:
          run_url: "${{ format('https://github.com/{0}/actions/runs/{1}', steps.get_head_ref.outputs.repo, github.run_id) }}"
          conclusion: "${{ steps.final_basic_validation.outcome == 'success' && 'success' || 'failure' }}"
        with:
          debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }}
          retries: 3
          # Default includes 422 which GitHub returns when it doesn't know about the head_sha we set the status for.
          # This occurs when the previous push succeeds, but the checks/pull request component of GitHub isn't yet aware
          # of the new commit. This isn't the common case, but it comes up enough to be annoying.
          retry-exempt-status-codes: 400, 401, 403, 404
          script: |
            // any JavaScript code can go here, you can use Node JS APIs too.
            // Docs: https://docs.github.com/en/rest/checks/runs#create-a-check-run
            // Rest: https://octokit.github.io/rest.js/v18#checks-create
            await github.rest.checks.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              head_sha: "${{ steps.get_head.outputs.HEAD }}",
              name: "Rule Tests and ID Updated",
              status: "completed",
              conclusion: process.env.conclusion,
              details_url: process.env.run_url,
              output: {
                title: "Rule Tests and ID Updated",
                summary: "Rule Tests and ID Updated",
                text: "Rule Tests and ID Updated",
              },
            });

      - name: Get changed detection-rules
        id: changed-files
        uses: tj-actions/changed-files@v39
        with:
          files: "detection-rules/**"
          recover_deleted_files: true

      - name: Checkout base
        uses: actions/checkout@v4
        if: ${{ steps.get_base_ref.outputs.run_all != 'true' }}
        with:
          ref: ${{ steps.get_base_ref.outputs.base_ref }}
          repository: sublime-security/sublime-rules
          depth: 0
          path: sr-main

      - name: Rename files in sr-main based on rule id
        if: ${{ steps.get_base_ref.outputs.run_all != 'true' }}
        run: |
          cd sr-main/detection-rules

          for file in *.yml
          do
            id=$(yq '.id' "$file")
            mv "$file" "${id}.yml"
          done


      - name: "Find updated rule IDs"
        id: find_ids
        run: |
          for file in detection-rules/*.yml; do
            rule_id=$(yq '.id' $file)
          
            if [[ "${{ steps.get_base_ref.outputs.run_all }}" == "true" ]]; then
              altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids")
              continue
            fi
          
            new_source=$(yq '.source' "$file")
            old_source=$(yq '.source' "sr-main/detection-rules/$rule_id.yml" || echo '')
            
            # We only need to care when rule source is changed. This will handle renames, tag changes, etc.
            if [[ "$new_source" != "$old_source" ]]; then
              echo "$file ($rule_id) has altered source"
              altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids")
            fi
          done
          
          for file in ${{ steps.changed-files.outputs.deleted_files }}; do
            rule_id=$(yq '.id' $file)
            echo "$file ($rule_id) was deleted"
            altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids")
          done
          
          echo "Altered Ruled IDs: [$altered_rule_ids]"
          echo "##[set-output name=rule_ids;]$(echo $altered_rule_ids)"
          # TODO: This doesn't solve for a modified rule_id. We could merge with any files known on 'main', but changing
          # a rule ID is a separate problem.

      - name: Get PR Number
        if: github.event_name == 'pull_request_target' || github.event_name == 'issue_comment'
        id: find_pr_number
        run: |
          if [[ "${{ github.event_name }}" == 'pull_request_target' || "${{ github.event_name }}" == 'merge_group' ]]; then
            result="${{ github.event.number }}"
          elif [[ "${{ github.event_name }}" == 'issue_comment' ]]; then
            result="${{ github.event.issue.number }}"
          fi

          echo "PR $result"
          echo "##[set-output name=result;]$result"


      - name: "Find mql-mimic-exempt Comments"
        uses: actions/github-script@v6
        id: find_emls_to_skip
        if: steps.find_pr_number.outputs.result != ''
        with:
          debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }}
          result-encoding: string
          script: |
            const opts = github.rest.issues.listComments.endpoint.merge({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: "${{ steps.find_pr_number.outputs.result }}",
            })
            const comments = await github.paginate(opts)
            
            const seperatorRegex = /[\s:,;\/]+/
            const exemptRegex = /\/mql-mimic-exempt((?:[\s:,;\/]+#*\d+)+)/gis

            let allEMLsToSkip = []
            for (const comment of comments) {
              if (comment.author_association !== "MEMBER") {
                console.log("Ignoring comment from non-member" + comment.user.login)
              }

              while ((m = exemptRegex.exec(comment.body)) !== null) {
                if (m.index === exemptRegex.lastIndex) {
                  break
                }
                  
                // The result can be accessed through the `m`-variable.
                m.forEach((match, groupIndex) => {
                  if (groupIndex != 1) {
                    return
                  }
            
                  console.log("Found MQL Mimic Exemption EMLs: " + match)
                  
                  // First cut out all (optional) #
                  match = match.replaceAll("#", "")
                  let emls = match.split(seperatorRegex)
                  console.log("Split EMLs: " + JSON.stringify(emls))
                  allEMLsToSkip = allEMLsToSkip.concat(emls.filter((s) => s !== ""))
                });
              }
            }
            
            console.log("All EMLs: " + JSON.stringify(allEMLsToSkip))
            // MQL Mimic will handle duplicates gracefully, no need to handle here.
            return allEMLsToSkip.join(" ")

      - name: "Find Existing MQL Mimic Test Results"
        uses: actions/github-script@v6
        id: find_mql_mimic_results
        if: ${{ github.event_name != 'merge_group' }}
        env:
          sha: '${{ steps.get_head.outputs.HEAD }}'
        with:
          debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }}
          script: |
            const result = await github.rest.checks.listForRef({
              check_name: "MQL Mimic Tests",
              owner: "sublime-security",
              repo: "sublime-rules",
              ref: process.env.sha
            })
            
            let existingRuns = result.data.check_runs.map((r) => r.id)
            console.log(existingRuns)
            return existingRuns

      - name: "Trigger MQL Mimic Tests"
        if: ${{ github.event_name != 'merge_group' }}
        env:
          trigger_url: '${{ secrets.MQL_MOCK_TRIGGER }}'
          branch: '${{ steps.get_head_ref.outputs.head_ref }}'
          repo: '${{ steps.get_head_ref.outputs.repo }}'
          token: '${{ secrets.GITHUB_TOKEN }}'
          sha: '${{ steps.get_head.outputs.HEAD }}'
          only_rule_ids: '${{ steps.find_ids.outputs.rule_ids }}'
          skip_eml_ids: '${{ steps.find_emls_to_skip.outputs.result }}'
        run: |
          body='{"branch":"'$branch'","repo":"'$repo'","token":"'$token'","sha":"'$sha'","only_rule_ids":"'$only_rule_ids'","skip_eml_ids":"'$skip_eml_ids'"}'
          echo $body
          
          curl -X POST $trigger_url  \
            -H 'Content-Type: application/json' \
            -d "$body"

      - name: Wait for MQL Mimic check to be completed
        if: ${{ github.event_name != 'merge_group' }}
        uses: sublime-security/action-wait-for-check@master
        # Wait for results so that the token remains valid while the test suite is executing and posting a check here.
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          checkName: "MQL Mimic Tests"
          ref: ${{ steps.get_head.outputs.HEAD }}
          timeoutSeconds: 3600
          ignoreIDs: ${{ steps.find_mql_mimic_results.outputs.result }}

      - name: "Create MQL Mimic Check"
        uses: actions/github-script@v6
        if: ${{ github.event_name == 'merge_group' }}
        id: create_check
        env:
          run_url: "${{ format('https://github.com/{0}/actions/runs/{1}', steps.get_head_ref.outputs.repo, github.run_id) }}"
        with:
          debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }}
          retries: 3
          # Default includes 422 which GitHub returns when it doesn't know about the head_sha we set the status for.
          # This occurs when the previous push succeeds, but the checks/pull request component of GitHub isn't yet aware
          # of the new commit. This isn't the common case, but it comes up enough to be annoying.
          retry-exempt-status-codes: 400, 401, 403, 404
          script: |
            // any JavaScript code can go here, you can use Node JS APIs too.
            // Docs: https://docs.github.com/en/rest/checks/runs#create-a-check-run
            // Rest: https://octokit.github.io/rest.js/v18#checks-create
            const response = await github.rest.checks.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              head_sha: "${{ steps.get_head.outputs.HEAD }}",
              name: "MQL Mimic Tests",
              status: "completed",
              conclusion: "success",
              details_url: process.env.run_url,
              output: {
                title: "MQL Mimic Tests",
                summary: "MQL Mimic tests are not run on merge queues",
                text: "MQL Mimic auto pass",
              },
            });
            
            return response["data"]["id"]