updated waf example

ansibleguy · May 25, 2024 · 179de51 · 179de51
1 parent 7940d74
commit 179de51
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 185 deletions.
diff --git a/ExampleWAF.md b/ExampleWAF.md
@@ -24,6 +24,7 @@ haproxy:
         # deny_dangerous_methods: true
         block_script_bots: true
         block_bad_crawler_bots: true
+        flag_bots: true
         block_script_kiddies: true
 
       routes:
@@ -102,36 +103,50 @@ root@test-ag-haproxy-waf:/# cat /etc/haproxy/conf.d/frontend.cfg
 >     mode http
 >     bind [::]:80 v4v6
 > 
+>  frontend fe_web
+>     mode http
+>     bind [::]:80 v4v6
+> 
 >     http-request deny status 405 default-errorfiles if !{ method HEAD GET POST }
 > 
 >     # block well-known script-bots
->     http-request deny status 425 default-errorfiles if { req.fhdr(User-Agent) -m sub -i curl wget Apache-HttpClient nmap Metasploit headless cypress go-http-client zgrab python httpx httpcore aiohttp httputil urllib GuzzleHttp phpcrawl Zend_Http_Client Wordpress Symfony-HttpClient cpp-httplib java perl axios ruby }
+>     http-request deny status 425 default-errorfiles if { req.fhdr(User-Agent) -m sub -i -f /etc/haproxy/lst/waf-badbot-ua-sub.lst }
 >     # block well-known bad-crawler-bots
->     http-request deny status 425 default-errorfiles if { req.fhdr(User-Agent) -m sub -i spider test-bot tiny-bot fidget-spinner-bot download scrapy }
+>     http-request deny status 425 default-errorfiles if { req.fhdr(User-Agent) -m str -i -f /etc/haproxy/lst/waf-crawler-ua-full.lst }
+>     http-request deny status 425 default-errorfiles if { req.fhdr(User-Agent) -m sub -i -f /etc/haproxy/lst/waf-crawler-ua-sub.lst }
 >     # block script-kiddy requests
->     http-request deny status 425 default-errorfiles if { path_beg -i /cgi-bin/ /icons/ /manager/ /php /program/ /pwd/ /shaAdmin/ /typo3/ /admin/ /dbadmin/ /db/ /solr/ /weaver/ /joomla/ /App/ /webdav/ /xmlrpc /% /. /securityRealm/ /magmi/ /menu/ /etc/ /HNAP1 }
->     http-request deny status 425 default-errorfiles if { path_end -i .php .asp .aspx .esp .lua .rsp .ashx .dll .bin .cgi .cs .application .exe .env .git/config .git/HEAD .git/index .DS_Store .aws/config .config .settings .tar .tgz .gz .bz2 .rar .7z .sql .sqlite3 .bak }
->     http-request deny status 425 default-errorfiles if { path_sub -i /../ }
+>     http-request deny status 425 default-errorfiles if { path -m beg -i -f /etc/haproxy/lst/waf-script-kiddy-path-beg.lst }
+>     http-request deny status 425 default-errorfiles if { path -m end -i -f /etc/haproxy/lst/waf-script-kiddy-path-end.lst }
+>     http-request deny status 425 default-errorfiles if { path -m sub -i -f /etc/haproxy/lst/waf-script-kiddy-path-sub.lst }
+>     # FLAG BOTS
+>     ## flag bots by common user-agent substrings
+>     http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m sub -i -f /etc/haproxy/lst/waf-bot-ua-sub.lst }
+> 
+>     ## unusual if action has no referrer; could produce false-positives in some special cases
+>     http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } !{ method GET HEAD } !{ req.hdr(Referer) -m found }
+>     ## browsers set these ones usually
+>     http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } !{ req.hdr(Accept-Language) -m found }
+>     http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } !{ req.fhdr(User-Agent) -m found }
+> 
+>     http-request set-var(txn.bot) int(0) if !{ var(txn.bot) -m found }
+>     http-request capture var(txn.bot) len 1
+> 
 >     # Security headers
 >     http-response set-header Strict-Transport-Security "max-age=16000000; includeSubDomains; preload;"
 >     http-response set-header X-Frame-Options "DENY"
 >     http-response set-header X-Content-Type-Options "nosniff"
 >     http-response set-header X-Permitted-Cross-Domain-Policies "none"
 >     http-response set-header X-XSS-Protection "1; mode=block"
 >     # SSL fingerprint
->     http-request set-var(txn.fp_ssl_p1) ssl_fc_cipherlist_bin(1),be2dec(-,2)
->     http-request set-var(txn.fp_ssl_p2) ssl_fc_extlist_bin(1),be2dec(-,2)
->     http-request set-var(txn.fp_ssl_p3) ssl_fc_eclist_bin(1),be2dec(-,2)
->     http-request set-var(txn.fp_ssl_p4) ssl_fc_ecformats_bin,be2dec(-,1)
->     http-request set-var(txn.fingerprint_ssl_raw) "ssl_fc_protocol_hello_id,concat(',',txn.fp_ssl_p1),concat(',',txn.fp_ssl_p2),concat(',',txn.fp_ssl_p3),concat(',',txn.fp_ssl_p4)"
->     http-request set-var(txn.fingerprint_ssl) var(txn.fingerprint_ssl_raw),digest(md5),hex,lower
+>     http-request set-header X-FINGERPRINT-JA3-RAW %[ssl_fc_protocol_hello_id],%[ssl_fc_cipherlist_bin(1),be2dec(-,2)],%[ssl_fc_extlist_bin(1),be2dec(-,2)],%[ssl_fc_eclist_bin(1),be2dec(-,2)],%[ssl_fc_ecformats_bin,be2dec(-,1)]
+>     http-request set-var(txn.fingerprint_ssl) req.fhdr(X-FINGERPRINT-JA3-RAW),digest(md5),hex,lower
 >     http-request capture var(txn.fingerprint_ssl) len 32
 > 
 >     http-request capture req.fhdr(User-Agent) len 200
 > 
 >     # BACKEND be_test
->     acl be_test_domains req.hdr(host) -m str -i app.test.ansibleguy.net
->     use_backend be_test if be_test_domains
+>     acl be_test_filter_domains req.hdr(host) -m str -i app.test.ansibleguy.net
+>     use_backend be_test if be_test_filter_domains
 > 
 >     default_backend be_fallback
 

diff --git a/molecule/default/converge.yml b/molecule/default/converge.yml
@@ -1,150 +1,5 @@
 ---
 
-- name: Converge HAProxy with basic config
-  hosts: test-ag-haproxy-base
-  module_defaults:
-    ansible.builtin.setup:
-      gather_subset: ['distribution']
-  gather_facts: true
-
-  vars:
-    no_prompts: true
-
-    haproxy:
-      stats:
-        enable: true
-
-      frontends:
-        fe_web:
-          bind: ['[::]:80 v4v6']
-
-          routes:
-            be_test1:
-              domains: 'app.test.ansibleguy.net'
-
-            be_test2:
-              domains: ['app1.test.ansibleguy.net', 'app2.test.ansibleguy.net']
-              filter_ip: ['192.168.0.0/16', '172.16.0.0/12', '10.0.0.0/8']
-              filter_not_ip: ['192.168.100.0/22', '10.50.0.0/16']
-
-          default_backend: 'be_fallback'
-
-      backends:
-        be_test1:
-          sticky: true
-          servers:
-            - 'srv1 192.168.10.11:80'
-            - 'srv2 192.168.10.12:80'
-
-        be_test2:
-          servers:
-            - 'srv3 192.168.10.11:80'
-            - 'srv4 192.168.10.12:80'
-
-        be_fallback:
-          lines: 'http-request redirect code 302 location https://github.com/ansibleguy'
-
-  roles:
-    - ansibleguy.infra_haproxy
-
-# NOTE: not testing actual certificate creation
-- name: Converge HAProxy with ACME config
-  hosts: test-ag-haproxy-acme
-  module_defaults:
-    ansible.builtin.setup:
-      gather_subset: ['distribution']
-  gather_facts: true
-
-  vars:
-    no_prompts: true
-
-    haproxy:
-      acme:
-        enable: true
-        email: '[email protected]'
-        ca: 'letsencrypt-test'
-
-      frontends:
-        fe_web:
-          bind: ['[::]:80 v4v6', '[::]:443 v4v6 ssl']
-          acme:
-            enable: true
-
-          routes:
-            be_test:
-
-          default_backend: 'be_fallback'
-
-      backends:
-        be_test:
-          servers:
-            - 'srv-1 192.168.10.11:80'
-            - 'srv-2 192.168.10.12:80'
-
-        be_fallback:
-          lines: 'http-request redirect code 302 location https://github.com/ansibleguy'
-
-  roles:
-    - ansibleguy.infra_haproxy
-
-- name: Converge HAProxy with GeoIP config
-  hosts: test-ag-haproxy-geoip
-  module_defaults:
-    ansible.builtin.setup:
-      gather_subset: ['distribution']
-  gather_facts: true
-
-  vars:
-    no_prompts: true
-
-    haproxy:
-      geoip:
-        enable: true
-        token: "{{ geoip_key.stdout }}"
-
-      frontends:
-        fe_web:
-          bind: ['[::]:80 v4v6']
-          geoip:
-            enable: true
-            country: true
-            asn: true
-            as_name: true
-
-          routes:
-            be_test1:
-              domains: ['app1.test.ansibleguy.net']
-              filter_country: 'AT'
-              filter_asn: '1337'
-
-            be_test2:
-              domains: ['app2.test.ansibleguy.net']
-              filter_not_country: ['CN', 'RU', 'US']
-              filter_not_asn: ['100000', '120000']
-
-          default_backend: 'be_fallback'
-
-      backends:
-        be_test1:
-          servers: 'srv1 192.168.10.11:80'
-
-        be_test2:
-          servers: 'srv2 192.168.10.12:80'
-
-        be_fallback:
-          lines: 'http-request redirect code 302 location https://github.com/ansibleguy'
-
-  pre_tasks:
-    - name: Loading GeoIP Token
-      ansible.builtin.command: "cat {{ lookup('ansible.builtin.env', 'HOME') }}/.secret/geoip.key"
-      delegate_to: localhost
-      become: false
-      register: geoip_key
-      changed_when: false
-      check_mode: false
-
-  roles:
-    - ansibleguy.infra_haproxy
 
 - name: Converge HAProxy with WAF config
   hosts: test-ag-haproxy-waf

diff --git a/tasks/debian/geoip.yml b/tasks/debian/geoip.yml
@@ -6,19 +6,6 @@
     shell: '/usr/sbin/nologin'
     comment: 'HAProxy GeoIP Lookup Serviceuser'
 
-- name: HAProxy | GeoIP | Create map & lua directory
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: directory
-    owner: 'root'
-    group: 'haproxy'
-    mode: 0750
-  loop:
-    - "{{ HAPROXY_HC.path.map }}"
-    - "{{ HAPROXY_HC.path.lst }}"
-    - "{{ HAPROXY_HC.path.lua }}"
-    - '/tmp/haproxy'
-
 - name: HAProxy | GeoIP | Touch map files
   ansible.builtin.file:
     path: "{{ item }}"

diff --git a/tasks/debian/install.yml b/tasks/debian/install.yml
@@ -52,13 +52,19 @@
     - haproxy_pkg.changed
     - no_prompts | bool or haproxy_pkg_prompt.user_input | bool
 
-- name: HAProxy | Install | Create config directory
+- name: HAProxy | Install | Create directories
   ansible.builtin.file:
-    path: "{{ HAPROXY_HC.path.config }}"
+    path: "{{ item }}"
     state: directory
     owner: 'root'
     group: 'haproxy'
     mode: 0750
+  loop:
+    - "{{ HAPROXY_HC.path.config }}"
+    - "{{ HAPROXY_HC.path.map }}"
+    - "{{ HAPROXY_HC.path.lst }}"
+    - "{{ HAPROXY_HC.path.lua }}"
+    - '/tmp/haproxy'
 
 - name: HAProxy | Install | Create service-override directory
   ansible.builtin.file:

diff --git a/templates/etc/haproxy/conf.d/inc/security.j2 b/templates/etc/haproxy/conf.d/inc/security.j2
@@ -20,7 +20,7 @@
     http-request deny status {{ HAPROXY_WAF.block_code }} {{ BLOCK_ERRORFILE }} if { req.fhdr(User-Agent) -m str -i -f {{ HAPROXY_HC.path.lst }}/{{ HAPROXY_HC.file.lst.crawler_full }} }
 {%   endif %}
 {%   if HAPROXY_WAF.user_agents.bad_crawlers.sub | length > 0 %}
-    http-request deny status {{ HAPROXY_WAF.block_code }} {{ BLOCK_ERRORFILE }} if { req.fhdr(User-Agent) -m sub -i -f {{ HAPROXY_HC.path.lst }}/{{ HAPROXY_HC.file.lst.crawler_sub}} }
+    http-request deny status {{ HAPROXY_WAF.block_code }} {{ BLOCK_ERRORFILE }} if { req.fhdr(User-Agent) -m sub -i -f {{ HAPROXY_HC.path.lst }}/{{ HAPROXY_HC.file.lst.crawler_sub }} }
 {%   endif %}
 {% endif %}
 {% if cnf.security.block_script_kiddies | bool %}
@@ -37,19 +37,19 @@
 {%   if not cnf.security.block_script_bots | bool %}
     ## flag well-known script-bots
 {%     if HAPROXY_WAF.user_agents.script.full | length > 0 %}
-    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m str -i {{ HAPROXY_WAF.user_agents.script.full | ensure_list | join(' ') }} }
+    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m str -i -f {{ HAPROXY_HC.path.lst }}/{{ HAPROXY_HC.file.lst.bad_bot_full }} }
 {%     endif %}
 {%     if HAPROXY_WAF.user_agents.script.sub | length > 0 %}
-    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m sub -i {{ HAPROXY_WAF.user_agents.script.sub | ensure_list | join(' ') }} }
+    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m sub -i -f {{ HAPROXY_HC.path.lst }}/{{ HAPROXY_HC.file.lst.bad_bot_sub }} }
 {%     endif %}
 {%   endif %}
 {%   if not cnf.security.block_bad_crawler_bots | bool %}
     ## flag well-known bad-crawler-bots
 {%     if HAPROXY_WAF.user_agents.bad_crawlers.full | length > 0 %}
-    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m str -i {{ HAPROXY_WAF.user_agents.bad_crawlers.full | ensure_list | join(' ') }} }
+    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m str -i -f {{ HAPROXY_HC.path.lst }}/{{ HAPROXY_HC.file.lst.crawler_full }} }
 {%     endif %}
 {%     if HAPROXY_WAF.user_agents.bad_crawlers.sub | length > 0 %}
-    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m sub -i {{ HAPROXY_WAF.user_agents.bad_crawlers.sub | ensure_list | join(' ') }} }
+    http-request set-var(txn.bot) int(1) if !{ var(txn.bot) -m found } { req.fhdr(User-Agent) -m sub -i -f {{ HAPROXY_HC.path.lst }}/{{ HAPROXY_HC.file.lst.crawler_sub }} }
 {%     endif %}
 {%   endif %}
     ## unusual if action has no referrer; could produce false-positives in some special cases

diff --git a/templates/etc/haproxy/conf.d/inc/security_only_fe.j2 b/templates/etc/haproxy/conf.d/inc/security_only_fe.j2
@@ -5,12 +5,8 @@
 {%   endfor %}
 {% endif %}
 {% if cnf.security.fingerprint_ssl | bool %}
-    # SSL fingerprint{# TODO: make more compact - did not find a better way of setting it as variable +#}
-    http-request set-var(txn.fp_ssl_p1) ssl_fc_cipherlist_bin(1),be2dec(-,2)
-    http-request set-var(txn.fp_ssl_p2) ssl_fc_extlist_bin(1),be2dec(-,2)
-    http-request set-var(txn.fp_ssl_p3) ssl_fc_eclist_bin(1),be2dec(-,2)
-    http-request set-var(txn.fp_ssl_p4) ssl_fc_ecformats_bin,be2dec(-,1)
-    http-request set-var(txn.fingerprint_ssl_raw) "ssl_fc_protocol_hello_id,concat(',',txn.fp_ssl_p1),concat(',',txn.fp_ssl_p2),concat(',',txn.fp_ssl_p3),concat(',',txn.fp_ssl_p4)"
-    http-request set-var(txn.fingerprint_ssl) var(txn.fingerprint_ssl_raw),digest(md5),hex,lower
+    # SSL fingerprint
+    http-request set-header X-FINGERPRINT-JA3-RAW %[ssl_fc_protocol_hello_id],%[ssl_fc_cipherlist_bin(1),be2dec(-,2)],%[ssl_fc_extlist_bin(1),be2dec(-,2)],%[ssl_fc_eclist_bin(1),be2dec(-,2)],%[ssl_fc_ecformats_bin,be2dec(-,1)]
+    http-request set-var(txn.fingerprint_ssl) req.fhdr(X-FINGERPRINT-JA3-RAW),digest(md5),hex,lower
     http-request capture var(txn.fingerprint_ssl) len 32
 {% endif %}