🤖 Add disallow for news and nyheter #2644 (#2645)

* 🤖 Add disallow for news and nyheter #2644 * Add search params #2644 * Change & ? * only search params #2644 * Change to wildcard * Add newline break #2644
equinor · Nov 6, 2024 · c8674f9 · c8674f9
1 parent cbc520f
commit c8674f9
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/web/pages/robots.txt.ts b/web/pages/robots.txt.ts
@@ -20,7 +20,7 @@ export const crawlableDomains = [
 
 const robots = (domain: string) => `User-agent: *
 ${crawlableDomains.includes(domain) ? 'Allow' : 'Disallow'}: /
-${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html' : ''}
+${domain === 'www.equinor.com' ? 'Disallow: /search \nDisallow: /search.html \nDisallow: /*?*' : ''}
 Sitemap: ${domain.startsWith('www') ? `https://${domain}` : domain}/sitemap.xml
 `