Merge pull request #520 from alphagov/update-robots-txt

Update robots.txt
alphagov · Jan 7, 2015 · 2500f32 · 2500f32
2 parents 067165c + 771d0ae
commit 2500f32
Showing 1 changed file with 3 additions and 4 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -9,12 +9,11 @@ Allow: /licence-finder
 Disallow: /business-finance-support-finder/*
 Allow: /business-finance-support-finder
 Disallow: /apply-for-a-licence
-# Don't allow indexing of experimental performance platform dashboards
-Disallow: /performance/experimental/*
 # Don't allow indexing of user needs pages
 Disallow: /info/*
 Sitemap: https://www.gov.uk/sitemap.xml
 Crawl-delay: 0.5
 
-user-agent: AhrefsBot
-disallow: /
+# https://ahrefs.com/robot/ crawls the site frequently
+User-agent: AhrefsBot
+Crawl-delay: 10