From f34ff6cf26a153d8724f9d6e12c4263c5e04da84 Mon Sep 17 00:00:00 2001 From: Jonathan S Date: Sat, 12 Aug 2023 23:38:19 -0400 Subject: [PATCH] AO3-6583 Updated the robots.txt to block chatGPT. (#4594) * AO3-6583 Updated the robots.txt to block chatGPT. * Adjusted robots per request of reviewer. --- public/robots.public.txt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/public/robots.public.txt b/public/robots.public.txt index f4ac6386c46..5ff24d7ec7e 100644 --- a/public/robots.public.txt +++ b/public/robots.public.txt @@ -1,5 +1,4 @@ # See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# User-agent: * Disallow: /works? # cruel but efficient @@ -12,7 +11,7 @@ Disallow: /people/search? Disallow: /tags/search? Disallow: /works/search? -User-agent: Googlebot +User-agent: Googlebot Disallow: /autocomplete/ Disallow: /downloads/ Disallow: /external_works/ @@ -22,17 +21,17 @@ Disallow: /*search? Disallow: /*?*query= Disallow: /*?*sort_ Disallow: /*?*selected_tags -Disallow: /*?*selected_pseuds -Disallow: /*?*use_openid Disallow: /*?*view_adult Disallow: /*?*tag_id Disallow: /*?*pseud_id Disallow: /*?*user_id Disallow: /*?*pseud= -Disallow: /people?*show= User-agent: CCBot Disallow: / +User-agent: GPTBot +Disallow: / + User-agent: Slurp Crawl-delay: 30