From 4408d67d35f204620019547567fb48c01aebc43a Mon Sep 17 00:00:00 2001 From: Tommy Date: Sun, 13 Oct 2024 13:46:37 -0700 Subject: [PATCH] Add robots handling --- etc/nginx/conf.d/sites_default_quic.conf | 3 ++- etc/nginx/conf.d/sites_miniflux.conf | 3 ++- etc/nginx/conf.d/sites_nextcloud.conf | 3 ++- etc/nginx/conf.d/sites_uptime-kuma.conf | 3 ++- etc/nginx/conf.d/sites_vaultwarden.conf | 3 ++- etc/nginx/snippets/robots.conf | 8 ++++++++ setup.sh | 2 ++ srv/nginx/robots.txt | 2 ++ 8 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 etc/nginx/snippets/robots.conf create mode 100644 srv/nginx/robots.txt diff --git a/etc/nginx/conf.d/sites_default_quic.conf b/etc/nginx/conf.d/sites_default_quic.conf index 1d1503a..8204c03 100644 --- a/etc/nginx/conf.d/sites_default_quic.conf +++ b/etc/nginx/conf.d/sites_default_quic.conf @@ -6,9 +6,10 @@ server { server_name hostname.of.your.server; - include snippets/universal_paths.conf; include snippets/hsts.conf; include snippets/quic.conf; + include snippets/robots.conf; + include snippets/universal_paths.conf; ssl_certificate /etc/letsencrypt/live/hostname.of.your.server/fullchain.pem; ssl_certificate_key /etc/letsencrypt/live/hostname.of.your.server/privkey.pem; diff --git a/etc/nginx/conf.d/sites_miniflux.conf b/etc/nginx/conf.d/sites_miniflux.conf index 13b39a8..285580d 100644 --- a/etc/nginx/conf.d/sites_miniflux.conf +++ b/etc/nginx/conf.d/sites_miniflux.conf @@ -10,12 +10,13 @@ server { ssl_certificate_key /etc/letsencrypt/live/miniflux.yourdomain.tld/privkey.pem; ssl_trusted_certificate /etc/letsencrypt/live/miniflux.yourdomain.tld/chain.pem; - include snippets/universal_paths.conf; include snippets/hsts.conf; include snippets/security.conf; include snippets/cross-origin-security.conf; include snippets/quic.conf; include snippets/proxy.conf; + include snippets/robots.conf; + include snippets/universal_paths.conf; proxy_hide_header Content-Security-Policy; add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; frame-src *; img-src *; manifest-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; base-uri 'none'; block-all-mixed-content; form-action 'self'; frame-ancestors 'self'; upgrade-insecure-requests"; diff --git a/etc/nginx/conf.d/sites_nextcloud.conf b/etc/nginx/conf.d/sites_nextcloud.conf index 42e5f57..7256189 100644 --- a/etc/nginx/conf.d/sites_nextcloud.conf +++ b/etc/nginx/conf.d/sites_nextcloud.conf @@ -10,11 +10,12 @@ server { ssl_certificate_key /etc/letsencrypt/live/cloud.yourdomain.tld/privkey.pem; ssl_trusted_certificate /etc/letsencrypt/live/cloud.yourdomain.tld/chain.pem; - include snippets/universal_paths.conf; include snippets/hsts.conf; include snippets/security.conf; include snippets/quic.conf; include snippets/proxy.conf; + include snippets/robots.conf; + include snippets/universal_paths.conf; add_header Cross-Origin-Resource-Policy "same-origin" always; add_header Cross-Origin-Opener-Policy "same-origin" always; diff --git a/etc/nginx/conf.d/sites_uptime-kuma.conf b/etc/nginx/conf.d/sites_uptime-kuma.conf index cddcdb3..741cb11 100644 --- a/etc/nginx/conf.d/sites_uptime-kuma.conf +++ b/etc/nginx/conf.d/sites_uptime-kuma.conf @@ -10,13 +10,14 @@ server { ssl_certificate_key /etc/letsencrypt/live/uptime.yourdomain.tld/privkey.pem; ssl_trusted_certificate /etc/letsencrypt/live/uptime.yourdomain.tld/chain.pem; - include snippets/universal_paths.conf; include snippets/hsts.conf; include snippets/security.conf; include snippets/cross-origin-security.conf; include snippets/quic.conf; include snippets/proxy.conf; proxy_hide_header Content-Security-Policy; + include snippets/universal_paths.conf; + add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; img-src 'self' data:; frame-src 'self'; manifest-src 'self'; object-src 'self'; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; base-uri 'none'; block-all-mixed-content; form-action 'none'; frame-ancestors 'self'; upgrade-insecure-requests"; location / { diff --git a/etc/nginx/conf.d/sites_vaultwarden.conf b/etc/nginx/conf.d/sites_vaultwarden.conf index cd9ae3a..cd7b078 100644 --- a/etc/nginx/conf.d/sites_vaultwarden.conf +++ b/etc/nginx/conf.d/sites_vaultwarden.conf @@ -10,12 +10,13 @@ server { ssl_certificate_key /etc/letsencrypt/live/vault.yourdomain.tld/privkey.pem; ssl_trusted_certificate /etc/letsencrypt/live/vault.yourdomain.tld/chain.pem; - include snippets/universal_paths.conf; include snippets/hsts.conf; include snippets/security.conf; include snippets/cross-origin-security.conf; include snippets/quic.conf; include snippets/proxy.conf; + include snippets/robots.conf; + include snippets/universal_paths.conf; location / { proxy_pass http://vaultwarden:8080; diff --git a/etc/nginx/snippets/robots.conf b/etc/nginx/snippets/robots.conf new file mode 100644 index 0000000..1629276 --- /dev/null +++ b/etc/nginx/snippets/robots.conf @@ -0,0 +1,8 @@ +proxy_hide_header X-Robots-Tag; +add_header X-Robots-Tag "noindex, nofollow" always; + +location = /robots.txt { + root /srv/nginx; + allow all; + access_log off; +} \ No newline at end of file diff --git a/setup.sh b/setup.sh index 8e57257..918f379 100644 --- a/setup.sh +++ b/setup.sh @@ -66,6 +66,7 @@ fi # Setup webroot for NGINX ## Explicitly using /var/srv here because SELinux does not follow symlinks sudo semanage fcontext -a -t httpd_sys_content_t "$(realpath /srv/nginx)(/.*)?" +unpriv curl -s https://raw.githubusercontent.com/GrapheneOS/infrastructure/main/srv/nginx/robots.txt | sudo tee /srv/nginx/robots.txt > /dev/null sudo mkdir -p /srv/nginx/.well-known/acme-challenge sudo chmod -R 755 /srv/nginx/.well-known/acme-challenge @@ -122,6 +123,7 @@ unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/quic.conf | sudo tee /etc/nginx/snippets/quic.conf > /dev/null unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/security.conf | sudo tee /etc/nginx/snippets/security.conf > /dev/null unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/cross-origin-security.conf | sudo tee /etc/nginx/snippets/cross-origin-security.conf > /dev/null +unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/robots.conf | sudo tee /etc/nginx/snippets/robots.conf > /dev/null unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/universal_paths.conf | sudo tee /etc/nginx/snippets/universal_paths.conf > /dev/null if [ "${ip_pinning}" = '0' ]; then diff --git a/srv/nginx/robots.txt b/srv/nginx/robots.txt new file mode 100644 index 0000000..77470cb --- /dev/null +++ b/srv/nginx/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / \ No newline at end of file