Merge branch 'main' into bfilar.nlu_signals

.github/workflows/clear-old-test-rules.yml

@@ -63,7 +63,7 @@ jobs:
           
             if [[ "$in_open_pr" = "false" ]]; then
                 rm $file
-                echo "Removed $pr_num" >> ../message.txt
+                echo "Removed $file_pr_num" >> ../message.txt
             fi
           done
           

.github/workflows/rule-validate.yml

@@ -8,13 +8,18 @@ on:
   workflow_dispatch: {}
 
 concurrency:
-  group: ${{ github.head_ref }}
+  # For pull_request_target workflows we want to use head_ref -- the branch triggering the workflow. Otherwise,
+  # use ref, which is the branch for a push event.
+  group: ${{ github.event_name == 'pull_request_target' && github.head_ref || github.ref }}
   cancel-in-progress: true
 
 jobs:
   tests:
     name: Run Rule Validation
     runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      checks: write
 
     steps:
       - name: Set up yq
@@ -26,6 +31,16 @@ jobs:
           ref: ${{ github.head_ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Add Rule IDs as Needed & Check for Duplicates
+        # Run before testing, just in case this could invalidate the rule itself
+        run: |
+          pip install -r scripts/generate-rule-ids/requirements.txt
+          python scripts/generate-rule-ids/main.py
+
       - name: Validate Rules
         run: |
           for f in *-rules/*.yml
@@ -58,82 +73,41 @@ jobs:
         run: |
           ! /bin/sh -c 'ls **/*.yaml'
 
-  rule-ids:
-    name: Add Rule IDs & Validate Uniqueness
-    runs-on: ubuntu-20.04
-
-    permissions:
-      contents: write
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.head_ref }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          fetch-depth: 0
-
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Add Rule IDs as Needed & Check for Duplicates
-        run: |
-          pip install -r scripts/generate-rule-ids/requirements.txt
-          python scripts/generate-rule-ids/main.py
-
       - name: Commit & Push Results, if needed
         run: |
+          rm response.txt
+          
           if [ -z "$(git status --porcelain)" ]; then 
             echo "No files changed, nothing to do"
             exit 0
           fi
           
           git config user.name 'ID Generator'
           git config user.email '[email protected]'
-          git add -A
+          git add **/*.yml
           git commit -m "Auto add rule ID"
           git push origin ${{ github.head_ref }}
 
-      - name: Get the head ref (eg branch)
-        id: get_head
-        run: git rev-parse HEAD > sha.txt
-
-      - name: Upload SHA for later job
-        uses: actions/upload-artifact@v3
-        with:
-          name: sha
-          path: sha.txt
-
-  # When the rule-ids job adds a commit, GitHub won't trigger actions on the auto commit. Various alternatives
-  # were explored, but all run into issues when dealing with forks (well we'll see if this does too).
-  set-checks:
-    name: Set GH Checks Status
-    runs-on: ubuntu-20.04
-    # Both must complete successfully. We assume that rule-ids does not break our rule validation (if we're not confident
-    # we can change the ordering.
-    needs: [rule-ids, tests]
-
-    permissions:
-      checks: write
-
-    steps:
-
-      - name: Download SHA from Previous Job
-        uses: actions/download-artifact@v3
-        with:
-          name: sha
-
-      - name: Get the head ref (eg branch)
+      - name: Get the head SHA
         id: get_head
-        run: echo "##[set-output name=HEAD;]$(cat sha.txt)"
+        run: echo "##[set-output name=HEAD;]$(git rev-parse HEAD)"
 
+      # When we add a commit, GitHub won't trigger actions on the auto commit, so we're missing a required check on the
+      # HEAD commit.
+      # Various alternatives were explored, but all run into issues when dealing with forks. This sets a "Check" for
+      # the latest commit, and we can depend on that as a required check.
       - name: "Create a check run"
         uses: actions/github-script@v6
+        if: github.event_name == 'pull_request_target'
         env:
           parameter_url: '${{ github.event.pull_request.html_url }}'
         with:
           debug: ${{ secrets.ACTIONS_STEP_DEBUG || false }}
+          retries: 3
+          # Default includes 422 which GitHub returns when it doesn't know about the head_sha we set the status for.
+          # This occurs when the previous push succeeds, but the checks/pull request component of GitHub isn't yet aware
+          # of the new commit. This isn't the common case, but it comes up enough to be annoying.
+          retry-exempt-status-codes: 400, 401, 403, 404
           script: |
             // any JavaScript code can go here, you can use Node JS APIs too.
             // Docs: https://docs.github.com/en/rest/checks/runs#create-a-check-run
@@ -147,8 +121,8 @@ jobs:
               conclusion: "success",
               details_url: process.env.parameter_url,
               output: {
-                title: "Rule Tests and ID Generation",
-                summary: "Rule Tests and ID Generation",
-                text: "Rule Tests and ID Generation",
+                title: "Rule Tests and ID Updated",
+                summary: "Rule Tests and ID Updated",
+                text: "Rule Tests and ID Updated",
               },
             });

.github/workflows/update-test-rules.yml

@@ -51,7 +51,7 @@ jobs:
         uses: lewagon/[email protected]
         with:
           ref: ${{ steps.comment-branch.outputs.head_sha }}
-          check-name: 'Run Rule Validation'
+          check-name: 'Rule Tests and ID Updated'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           wait-interval: 10
 

detection-rules/attachment_adobe_image_lure_fts.yml

@@ -1,6 +1,5 @@
 name: "Attachment: Adobe image lure with suspicious link from first time sender"
-description: "Detects Adobe phishing messages with an Adobe logo attached, with suspicious link language
-  from a first-time sender."
+description: "Detects Adobe phishing messages with an Adobe logo attached, with suspicious link language from a first-time sender."
 type: "rule"
 severity: "medium"
 source: |
@@ -25,8 +24,15 @@ source: |
              and sender.email.domain.domain not in $sender_domains
           )
   )
-tags:
-  - "Suspicious link"
-  - "Brand impersonation"
+attack_types:
+  - "Malware/Ransomware"
+tactics_and_techniques:
+  - "Image as content"
+  - "Impersonation: Brand"
+detection_methods:
+  - "Content analysis"
   - "Computer Vision"
+  - "Optical Character Recognition"
+  - "Sender analysis"
+  - "URL analysis"
 id: "1d7add81-9822-576a-bcae-c4440e75e393"

detection-rules/attachment_any_html_in_archive_unsolicited.yml

@@ -25,6 +25,14 @@ source: |
           )
   )
 tags:
-  - "Suspicious attachment"
   - "Attack surface reduction"
+attack_types:
+  - "Credential Phishing"
+  - "Malware/Ransomware"
+tactics_and_techniques:
+  - "Evasion"
+  - "HTML smuggling"
+detection_methods:
+  - "Archive analysis"
+  - "File analysis"
 id: "6a67c02c-d405-531e-850a-1722849c5fe4"

detection-rules/attachment_any_html_new_sender.yml

@@ -27,7 +27,10 @@ source: |
           )
   )
 tags:
-  - "Suspicious attachment"
   - "Attack surface reduction"
+tactics_and_techniques:
   - "HTML smuggling"
+detection_methods:
+  - "HTML analysis"
+  - "Sender analysis"
 id: "57a8f5c5-c4c4-5268-b452-e381dc64ea42"

detection-rules/attachment_any_html_unsolicited.yml

@@ -27,7 +27,11 @@ source: |
       )
   )
 tags:
-  - "Suspicious attachment"
   - "Attack surface reduction"
+tactics_and_techniques:
   - "HTML smuggling"
+detection_methods:
+  - "File analysis"
+  - "HTML analysis"
+  - "Sender analysis"
 id: "ef36763f-917d-5338-b1ac-84047334dce8"

detection-rules/attachment_archive_with_chm.yml

@@ -14,7 +14,11 @@ source: |
   and any(attachments, .file_extension in~ $file_extensions_common_archives
       and any(file.explode(.), .file_extension =~ "chm")
   )
-tags:
-  - "Suspicious attachment"
-  - "Malware"
+attack_types:
+  - "Malware/Ransomware"
+tactics_and_techniques:
+  - "Evasion"
+detection_methods:
+  - "Archive analysis"
+  - "File analysis"
 id: "5280e94d-592b-5d80-b548-b0e2ad4c9eae"

detection-rules/attachment_archive_with_exe.yml

@@ -19,7 +19,12 @@ source: |
           any(.flavors.yara, . == "mz_file")
       )
   )
-tags:
-  - "Suspicious attachment"
-  - "Malware"
+attack_types:
+  - "Malware/Ransomware"
+tactics_and_techniques:
+  - "Evasion"
+detection_methods:
+  - "Archive analysis"
+  - "File analysis"
+  - "YARA"
 id: "e2b0ad86-1264-567e-b7c4-78d7ddbe7722"

detection-rules/attachment_archive_with_pdf_wsf_txt_attached.qakbot.yml

@@ -15,6 +15,13 @@ source: |
         and any(file.explode(.), .depth == 1 and .file_extension == "wsf")
   )
 tags:
-  - "QakBot"
-  - "Suspicious attachment"
+  - "Malfam: QakBot"
+attack_types:
+  - "Malware/Ransomware"
+tactics_and_techniques:
+  - "Evasion"
+  - "PDF"
+detection_methods:
+  - "Archive analysis"
+  - "File analysis"
 id: "16b2e239-a7d1-51ef-aac1-562b3cc59787"

detection-rules/attachment_callback_phish_with_img.yml

@@ -19,7 +19,7 @@ source: |
             and sender.email.domain.domain not in $recipient_domains
         )
   )
-  
+
   and sender.email.domain.root_domain in $free_email_providers
   and any(attachments, .file_extension in~ ('png', 'jpg', 'jpeg')
       and any(file.explode(.),
@@ -39,7 +39,7 @@ source: |
                 "*refund*",
                 "*+1*"
             ))) >= 4)
-  
+
       and any(file.explode(.),
           length(filter(.scan.strings.strings, strings.ilike(.,
                 "*geek squad*",
@@ -52,7 +52,18 @@ source: |
                 "*secure anywhere*"
             ))) >=1)
             )
-tags:
+attack_types:
   - "Callback Phishing"
-  - "Customer service fraud"
+tactics_and_techniques:
+  - "Evasion"
+  - "Free email provider"
+  - "Out of band pivot"
+  - "Social engineering"
+  - "Image as content"
+detection_methods:
+  - "Content analysis"
+  - "Optical Character Recognition"
+  - "Sender analysis"
+  - "URL analysis"
+  - "Computer Vision"
 id: "60acbb36-8ed1-562e-8027-260c2fdf0f04"

detection-rules/attachment_callback_phish_with_pdf.yml

@@ -23,7 +23,7 @@ source: |
 
   // sender is freemail
   and sender.email.domain.root_domain in $free_email_providers
-  
+
   // the attachment is a pdf with 1 page, and at least 60 ocr chars
   and any(attachments, .file_extension == "pdf"
       and any(file.explode(.),
@@ -47,7 +47,7 @@ source: |
                 "*refund*",
                 "*+1*"
             ))) >= 4)
-  
+
       // 1 of the following strings is found, representing common Callback brands          
       and any(file.explode(.),
           length(filter(.scan.strings.strings, strings.ilike(.,
@@ -60,7 +60,17 @@ source: |
                 "*paypal*",
             ))) >=1)
             )
-tags:
+attack_types:
   - "Callback Phishing"
-  - "Customer service fraud"
+tactics_and_techniques:
+  - "Evasion"
+  - "Free email provider"
+  - "Out of band pivot"
+  - "PDF"
+  - "Social engineering"
+detection_methods:
+  - "Exif analysis"
+  - "File analysis"
+  - "Optical Character Recognition"
+  - "Sender analysis"
 id: "ac33f097-af20-554c-b29a-56f21be1b285"

detection-rules/attachment_cve_2023_21716_rtf_fonts.yml

@@ -21,7 +21,11 @@ source: |
           and length(filter(.scan.strings.strings, strings.ilike(.,'{\f*;}'))) > 10000
       )
   )
-tags: 
-  - "Office exploit"
-  - "Suspicious attachment"
+attack_types:
+  - "Malware/Ransomware"
+tactics_and_techniques:
+  - "Exploit"
+detection_methods:
+  - "Content analysis"
+  - "File analysis"
 id: "23714cca-f0c6-5fa5-a68b-98c9d60a508d"

detection-rules/attachment_disallowed_file_type_in_archive.yml

@@ -194,6 +194,12 @@ source: |
       )
   )
 tags:
-  - "Suspicious attachment"
   - "Attack surface reduction"
+attack_types:
+  - "Malware/Ransomware"
+tactics_and_techniques:
+  - "Evasion"
+detection_methods:
+  - "Archive analysis"
+  - "File analysis"
 id: "3859e3e7-51c9-5259-9b7d-f8c0957696c0"