From f7f64a44acec8933ede9de822231978fcb062691 Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Wed, 22 Feb 2023 08:41:17 +0000 Subject: [PATCH] Allow whole site indexing, for ukwa/ukwa-services#96 --- static/robots.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/static/robots.txt b/static/robots.txt index 8af963213a..d9cb9019f6 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -9,7 +9,7 @@ Disallow: /datasets/ Disallow: /shine/search # Allow search engines to index specific sites: # As requested in https://github.com/ukwa/ukwa-services/issues/96 -Allow: /wayback/archive/*/http://www.europeandialogue.org/ +Allow: /wayback/archive/*/http://www.europeandialogue.org/* # Allow Twitterbot so social cards work: User-agent: Twitterbot