TEST - Update headers_russia_return_path.yml #2910
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rules PR CI | |
| on: | |
| push: | |
| branches: [ "main", "test-rules" ] | |
| pull_request_target: | |
| branches: [ "**" ] | |
| workflow_dispatch: {} | |
| concurrency: | |
| # For pull_request_target workflows we want to use head_ref -- the branch triggering the workflow. Otherwise, | |
| # use ref, which is the branch for a push event. | |
| group: ${{ github.event_name == 'pull_request_target' && github.head_ref || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request_target' }} | |
| jobs: | |
| tests: | |
| name: Run Rule Validation | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: write | |
| checks: write | |
| steps: | |
| - name: Set up yq | |
| uses: mikefarah/[email protected] | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| depth: 0 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Add Rule IDs as Needed & Check for Duplicates | |
| # Run before testing, just in case this could invalidate the rule itself | |
| run: | | |
| pip install -r scripts/generate-rule-ids/requirements.txt | |
| python scripts/generate-rule-ids/main.py | |
| - name: Validate Rules | |
| run: | | |
| for f in *-rules/*.yml | |
| do | |
| echo "Processing $f" | |
| http_code=$(yq -o=json eval 'del(.type)' "$f" | curl -H "Content-Type: application/json" -X POST --data-binary @- -o response.txt -w "%{http_code}" --silent https://playground.sublimesecurity.com/v1/rules/validate) | |
| echo '' >> response.txt | |
| cat response.txt | |
| if [[ "$http_code" != "200" ]]; then | |
| echo "Unexpected response $http_code" | |
| exit 1 | |
| fi | |
| done | |
| - name: Validate Insights and Signals | |
| run: | | |
| for f in {insights,signals}/**/*.yml | |
| do | |
| echo "Processing $f" | |
| http_code=$(yq eval 'del(.type) | .source = "length([\n\n" + .source + "\n]) >= 0"' "$f" -o=json | curl -H "Content-Type: application/json" -X POST --data-binary @- -o response.txt -w "%{http_code}" --silent https://playground.sublimesecurity.com/v1/rules/validate) | |
| echo '' >> response.txt | |
| cat response.txt | |
| if [[ "$http_code" != "200" ]]; then | |
| echo "Unexpected response $http_code" | |
| exit 1 | |
| fi | |
| done | |
| - name: Verify no .yaml files exist | |
| run: | | |
| ! /bin/sh -c 'ls **/*.yaml' | |
| - name: Commit & Push Results, if needed | |
| run: | | |
| rm response.txt | |
| if [ -z "$(git status --porcelain)" ]; then | |
| echo "No files changed, nothing to do" | |
| exit 0 | |
| fi | |
| git config user.name 'ID Generator' | |
| git config user.email '[email protected]' | |
| git add **/*.yml | |
| git commit -m "Auto add rule ID" | |
| git push origin ${{ github.head_ref }} | |
| - name: Get the head SHA | |
| id: get_head | |
| run: echo "##[set-output name=HEAD;]$(git rev-parse HEAD)" | |
| - name: Get changed detection-rules | |
| id: changed-files | |
| uses: tj-actions/changed-files@v39 | |
| with: | |
| files: "detection-rules/**" | |
| recover_deleted_files: true | |
| - name: Get base ref | |
| id: get_base_ref | |
| run: | | |
| if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then | |
| # Detect changes based on whatever we're merging into. | |
| echo "##[set-output name=ref;]${{ github.base_ref }}" | |
| elif [[ "${{ github.event_name }}" == 'push' ]]; then | |
| # Detect changes based on the previous commit | |
| echo "##[set-output name=ref;]$(git rev-parse HEAD^)" | |
| elif [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then | |
| # Run on a target, so run for all rules. | |
| echo "##[set-output name=run_all;]true" | |
| fi | |
| - name: Checkout base | |
| uses: actions/checkout@v4 | |
| if: ${{ steps.get_base_ref.outputs.run_all != 'true' }} | |
| with: | |
| ref: ${{ steps.get_base_ref.outputs.ref }} | |
| repository: sublime-security/sublime-rules | |
| depth: 0 | |
| path: sr-main | |
| - name: Rename files in sr-main based on rule id | |
| if: ${{ steps.get_base_ref.outputs.run_all != 'true' }} | |
| run: | | |
| cd sr-main/detection-rules | |
| for file in *.yml | |
| do | |
| id=$(yq '.id' "$file") | |
| mv "$file" "${id}.yml" | |
| done | |
| - name: "Find updated rule IDs" | |
| id: find_ids | |
| run: | | |
| for file in detection-rules/*.yml; do | |
| rule_id=$(yq '.id' $file) | |
| if [[ "${{ steps.get_base_ref.outputs.run_all }}" == "true" ]]; then | |
| altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids") | |
| continue | |
| fi | |
| new_source=$(yq '.source' "$file") | |
| old_source=$(yq '.source' "sr-main/detection-rules/$rule_id.yml" || echo '') | |
| # We only need to care when rule source is changed. This will handle renames, tag changes, etc. | |
| if [[ "$new_source" != "$old_source" ]]; then | |
| echo "$file ($rule_id) has altered source" | |
| altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids") | |
| fi | |
| done | |
| for file in ${{ steps.changed-files.outputs.deleted_files }}; do | |
| rule_id=$(yq '.id' $file) | |
| echo "$file ($rule_id) was deleted" | |
| altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids") | |
| done | |
| echo "Altered Ruled IDs: [$altered_rule_ids]" | |
| echo "##[set-output name=rule_ids;]$(echo $altered_rule_ids)" | |
| # TODO: This doesn't solve for a modified rule_id. We could merge with any files known on 'main', but changing | |
| # a rule ID is a separate problem. | |
| - name: "Trigger MQL Mimic Tests" | |
| env: | |
| trigger_url: '${{ secrets.MQL_MOCK_TRIGGER }}' | |
| branch: ${{ github.event_name == 'pull_request_target' && github.head_ref || github.ref }} | |
| repo: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name || github.repository }} | |
| token: '${{ secrets.GITHUB_TOKEN }}' | |
| sha: '${{ steps.get_head.outputs.HEAD }}' | |
| only_rule_ids: '${{ steps.find_ids.outputs.rule_ids }}' | |
| run: | | |
| body='{"branch":"'$branch'","repo":"'$repo'","token":"'$token'","sha":"'$sha'","only_rule_ids":"'$only_rule_ids'"}' | |
| echo $body | |
| curl -X POST $trigger_url \ | |
| -H 'Content-Type: application/json' \ | |
| -d "$body" | |
| # When we add a commit, GitHub won't trigger actions on the auto commit, so we're missing a required check on the | |
| # HEAD commit. | |
| # Various alternatives were explored, but all run into issues when dealing with forks. This sets a "Check" for | |
| # the latest commit, and we can depend on that as a required check. | |
| - name: "Create a check run" | |
| uses: actions/github-script@v6 | |
| if: github.event_name == 'pull_request_target' | |
| env: | |
| parameter_url: '${{ github.event.pull_request.html_url }}' | |
| with: | |
| debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }} | |
| retries: 3 | |
| # Default includes 422 which GitHub returns when it doesn't know about the head_sha we set the status for. | |
| # This occurs when the previous push succeeds, but the checks/pull request component of GitHub isn't yet aware | |
| # of the new commit. This isn't the common case, but it comes up enough to be annoying. | |
| retry-exempt-status-codes: 400, 401, 403, 404 | |
| script: | | |
| // any JavaScript code can go here, you can use Node JS APIs too. | |
| // Docs: https://docs.github.com/en/rest/checks/runs#create-a-check-run | |
| // Rest: https://octokit.github.io/rest.js/v18#checks-create | |
| await github.rest.checks.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| head_sha: "${{ steps.get_head.outputs.HEAD }}", | |
| name: "Rule Tests and ID Updated", | |
| status: "completed", | |
| conclusion: "success", | |
| details_url: process.env.parameter_url, | |
| output: { | |
| title: "Rule Tests and ID Updated", | |
| summary: "Rule Tests and ID Updated", | |
| text: "Rule Tests and ID Updated", | |
| }, | |
| }); | |
| - name: Wait for MQL Mimic check to be completed | |
| uses: fountainhead/[email protected] | |
| id: wait-for-build | |
| # Wait for results so that the token remains valid while the test suite is executing and posting a check here. | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| checkName: "MQL Mimic Tests" | |
| ref: ${{ steps.get_head.outputs.HEAD }} | |
| timeoutSeconds: 3600 | |