extended bot list

ansibleguy · May 22, 2024 · d912885 · d912885
1 parent a5783f7
commit d912885
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/defaults/main/2_waf.yml b/defaults/main/2_waf.yml
@@ -21,7 +21,8 @@ defaults_waf:
         # cli tools
         - 'curl'
         - 'wget'
-        - 'Apache-HttpClient'
+        - 'Apache'
+        - 'OpenSSL'
         - 'nmap'
         - 'Metasploit'
         # automation tools
@@ -33,6 +34,7 @@ defaults_waf:
         - 'go-http-client'
         - 'zgrab'
         - 'grpc-go'
+        - 'colly'
         # python
         - 'python'
         - 'httpx'
@@ -73,6 +75,8 @@ defaults_waf:
         - 'Metasploit'
         - 'ImageVacuum'
         - 'ELinks'
+        - 'Mozilla/3.0'
+        - 'Mozilla/4.0'
         # python
         - 'scrapy'
         # golang
@@ -108,6 +112,8 @@ defaults_waf:
         # duck duck go
         - 'DuckDuckBot'
         - 'DuckDuckGo'
+        # yahoo
+        - 'Yahoo'
 
     any:
       - 'bot'

diff --git a/templates/etc/haproxy/conf.d/inc/security.j2 b/templates/etc/haproxy/conf.d/inc/security.j2
@@ -52,7 +52,6 @@
     http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m sub -i {{ HAPROXY_WAF.user_agents.bad_crawlers.sub | ensure_list | join(' ') }} }
 {%     endif %}
 {%   endif %}
-    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m sub -i {{ HAPROXY_WAF.user_agents.search_engines.sub | ensure_list | join(' ') }} }
     ## unusual if action has no referrer; could produce false-positives in some special cases
     http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } !{ method GET HEAD } !{ req.hdr(Referer) -m found }
     ## browsers set these ones usually