Add robots handling

TommyTran732 · Oct 13, 2024 · 4408d67 · 4408d67
1 parent ba55409
commit 4408d67
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 5 deletions.
diff --git a/etc/nginx/conf.d/sites_default_quic.conf b/etc/nginx/conf.d/sites_default_quic.conf
@@ -6,9 +6,10 @@ server {
 
     server_name hostname.of.your.server;
 
-    include snippets/universal_paths.conf;
     include snippets/hsts.conf;
     include snippets/quic.conf;
+    include snippets/robots.conf;
+    include snippets/universal_paths.conf;
 
     ssl_certificate /etc/letsencrypt/live/hostname.of.your.server/fullchain.pem;
     ssl_certificate_key /etc/letsencrypt/live/hostname.of.your.server/privkey.pem;

diff --git a/etc/nginx/conf.d/sites_miniflux.conf b/etc/nginx/conf.d/sites_miniflux.conf
@@ -10,12 +10,13 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/miniflux.yourdomain.tld/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/miniflux.yourdomain.tld/chain.pem;
 
-    include snippets/universal_paths.conf;
     include snippets/hsts.conf;
     include snippets/security.conf;
     include snippets/cross-origin-security.conf;
     include snippets/quic.conf;
     include snippets/proxy.conf;
+    include snippets/robots.conf;
+    include snippets/universal_paths.conf;
 
     proxy_hide_header Content-Security-Policy;
     add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; frame-src *; img-src *; manifest-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; base-uri 'none'; block-all-mixed-content; form-action 'self'; frame-ancestors 'self'; upgrade-insecure-requests";

diff --git a/etc/nginx/conf.d/sites_nextcloud.conf b/etc/nginx/conf.d/sites_nextcloud.conf
@@ -10,11 +10,12 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/cloud.yourdomain.tld/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/cloud.yourdomain.tld/chain.pem;
 
-    include snippets/universal_paths.conf;
     include snippets/hsts.conf;
     include snippets/security.conf;
     include snippets/quic.conf;
     include snippets/proxy.conf;
+    include snippets/robots.conf;
+    include snippets/universal_paths.conf;
 
     add_header Cross-Origin-Resource-Policy "same-origin" always;
     add_header Cross-Origin-Opener-Policy "same-origin" always;

diff --git a/etc/nginx/conf.d/sites_uptime-kuma.conf b/etc/nginx/conf.d/sites_uptime-kuma.conf
@@ -10,13 +10,14 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/uptime.yourdomain.tld/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/uptime.yourdomain.tld/chain.pem;
 
-    include snippets/universal_paths.conf;
     include snippets/hsts.conf;
     include snippets/security.conf;
     include snippets/cross-origin-security.conf;
     include snippets/quic.conf;
     include snippets/proxy.conf;
     proxy_hide_header Content-Security-Policy;
+    include snippets/universal_paths.conf;
+
     add_header Content-Security-Policy "default-src 'none'; connect-src 'self'; img-src 'self' data:; frame-src 'self'; manifest-src 'self'; object-src 'self'; script-src 'self' 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; base-uri 'none'; block-all-mixed-content; form-action 'none'; frame-ancestors 'self'; upgrade-insecure-requests";
 
     location / {

diff --git a/etc/nginx/conf.d/sites_vaultwarden.conf b/etc/nginx/conf.d/sites_vaultwarden.conf
@@ -10,12 +10,13 @@ server {
     ssl_certificate_key /etc/letsencrypt/live/vault.yourdomain.tld/privkey.pem;
     ssl_trusted_certificate /etc/letsencrypt/live/vault.yourdomain.tld/chain.pem;
 
-    include snippets/universal_paths.conf;
     include snippets/hsts.conf;
     include snippets/security.conf;
     include snippets/cross-origin-security.conf;
     include snippets/quic.conf;
     include snippets/proxy.conf;
+    include snippets/robots.conf;
+    include snippets/universal_paths.conf;
 
     location / {
         proxy_pass http://vaultwarden:8080;

diff --git a/etc/nginx/snippets/robots.conf b/etc/nginx/snippets/robots.conf
@@ -0,0 +1,8 @@
+proxy_hide_header X-Robots-Tag;
+add_header X-Robots-Tag "noindex, nofollow" always;
+
+location = /robots.txt {
+    root /srv/nginx;
+    allow all;
+    access_log off;
+}
diff --git a/setup.sh b/setup.sh
@@ -66,6 +66,7 @@ fi
 # Setup webroot for NGINX
 ## Explicitly using /var/srv here because SELinux does not follow symlinks
 sudo semanage fcontext -a -t httpd_sys_content_t "$(realpath /srv/nginx)(/.*)?"
+unpriv curl -s https://raw.githubusercontent.com/GrapheneOS/infrastructure/main/srv/nginx/robots.txt | sudo tee /srv/nginx/robots.txt > /dev/null
 sudo mkdir -p /srv/nginx/.well-known/acme-challenge
 sudo chmod -R 755 /srv/nginx/.well-known/acme-challenge
 
@@ -122,6 +123,7 @@ unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main
 unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/quic.conf | sudo tee /etc/nginx/snippets/quic.conf > /dev/null
 unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/security.conf | sudo tee /etc/nginx/snippets/security.conf > /dev/null
 unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/cross-origin-security.conf | sudo tee /etc/nginx/snippets/cross-origin-security.conf > /dev/null
+unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/robots.conf | sudo tee /etc/nginx/snippets/robots.conf > /dev/null
 unpriv curl -s https://raw.githubusercontent.com/TommyTran732/NGINX-Configs/main/etc/nginx/snippets/universal_paths.conf | sudo tee /etc/nginx/snippets/universal_paths.conf > /dev/null
 
 if [ "${ip_pinning}" = '0' ]; then

diff --git a/srv/nginx/robots.txt b/srv/nginx/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /