diff --git a/.github/workflows/rule-validate.yml b/.github/workflows/rule-validate.yml
index dc15f63eeba..c98120c8aba 100644
--- a/.github/workflows/rule-validate.yml
+++ b/.github/workflows/rule-validate.yml
@@ -11,7 +11,7 @@ concurrency:
   # For pull_request_target workflows we want to use head_ref -- the branch triggering the workflow. Otherwise,
   # use ref, which is the branch for a push event.
   group: ${{ github.event_name == 'pull_request_target' && github.head_ref || github.ref }}
-  cancel-in-progress: true
+  cancel-in-progress: ${{ github.event_name == 'pull_request_target' }}
 
 jobs:
   tests:
@@ -26,7 +26,7 @@ jobs:
         uses: mikefarah/yq@v4.27.3
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
@@ -77,12 +77,12 @@ jobs:
       - name: Commit & Push Results, if needed
         run: |
           rm response.txt
-          
-          if [ -z "$(git status --porcelain)" ]; then 
+
+          if [ -z "$(git status --porcelain)" ]; then
             echo "No files changed, nothing to do"
             exit 0
           fi
-          
+
           git config user.name 'ID Generator'
           git config user.email 'hello@sublimesecurity.com'
           git add **/*.yml
@@ -100,14 +100,65 @@ jobs:
           files: "detection-rules/**"
           recover_deleted_files: true
 
+      - name: Get base ref
+        id: get_base_ref
+        run: |
+          if [[ "${{ github.event_name }}" == 'pull_request_target' ]]; then
+            # Detect changes based on whatever we're merging into.
+            echo "##[set-output name=ref;]${{ github.base_ref }}"
+          elif [[ "${{ github.event_name }}" == 'push' ]]; then
+            # Detect changes based on the previous commit
+            echo "##[set-output name=ref;]$(git rev-parse HEAD^)"
+          elif [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then
+            # Run on a target, so run for all rules.
+            echo "##[set-output name=run_all;]true"
+          fi
+
+      - name: Checkout base
+        uses: actions/checkout@v4
+        if: ${{ steps.get_base_ref.outputs.run_all != 'true' }}
+        with:
+          ref: ${{ steps.get_base_ref.outputs.ref }}
+          repository: sublime-security/sublime-rules
+          depth: 0
+          path: sr-main
+
+      - name: Rename files in sr-main based on rule id
+        if: ${{ steps.get_base_ref.outputs.run_all != 'true' }}
+        run: |
+          cd sr-main/detection-rules
+
+          for file in *.yml
+          do
+            id=$(yq '.id' "$file")
+            mv "$file" "${id}.yml"
+          done
+
+
       - name: "Find updated rule IDs"
         id: find_ids
         run: |
-          for file in ${{ steps.changed-files.outputs.all_changed_and_modified_files }}; do
-            echo "$file was changed"
+          for file in detection-rules/*.yml; do
             rule_id=$(yq '.id' $file)
           
-            echo "$file has rule ID $rule_id"
+            if [[ "${{ steps.get_base_ref.outputs.run_all }}" == "true" ]]; then
+              altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids")
+              continue
+            fi
+          
+            new_source=$(yq '.source' "$file")
+            old_source=$(yq '.source' "sr-main/detection-rules/$rule_id.yml" || echo '')
+            
+            # We only need to care when rule source is changed. This will handle renames, tag changes, etc.
+            if [[ "$new_source" != "$old_source" ]]; then
+              echo "$file ($rule_id) has altered source"
+              altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids")
+            fi
+          done
+          
+          for file in ${{ steps.changed-files.outputs.deleted_files }}; do
+            rule_id=$(yq '.id' $file)
+            echo "$file ($rule_id) was deleted"
             altered_rule_ids=$(echo "$rule_id"" ""$altered_rule_ids")
           done