From ef09e6ce393577fcc58691bf58885eeb740edbe9 Mon Sep 17 00:00:00 2001 From: Aminda Suomalainen Date: Mon, 29 Jan 2024 10:11:21 +0200 Subject: [PATCH] robots.txt: update from darkvisitors --- robots.txt | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/robots.txt b/robots.txt index 8f6aefe4..2243ce13 100644 --- a/robots.txt +++ b/robots.txt @@ -25,27 +25,31 @@ Noindex: /n/ Sitemap: https://aminda.eu/sitemap.xml Sitemap: https://aminda.eu/sitemaps.xml -# https://darkvisitors.com/robots-txt-builder -# Dark Visitors robots.txt - -# AI Assistant -# https://darkvisitors.com/agents/chatgpt-user - +# These were previously part of darkvisitors but are not anymore for some reason? User-agent: ChatGPT-User Disallow: / -# AI Assistant -# https://darkvisitors.com/agents/cohere-ai - User-agent: cohere-ai Disallow: / +User-agent: omgilibot +Disallow: / + +# https://darkvisitors.com/robots-txt-builder +# Dark Visitors robots.txt + # AI Data Scraper # https://darkvisitors.com/agents/anthropic-ai User-agent: anthropic-ai Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/bytespider + +User-agent: Bytespider +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/ccbot @@ -71,10 +75,7 @@ User-agent: GPTBot Disallow: / # AI Data Scraper -# https://darkvisitors.com/agents/omgilibot - -User-agent: omgilibot -Disallow: / +# https://darkvisitors.com/agents/omgili User-agent: omgili Disallow: /