From bf04b1fffdda3fd0c00e6a547a98d4d9df839260 Mon Sep 17 00:00:00 2001 From: Evan Date: Thu, 4 Jun 2020 14:34:43 -0400 Subject: [PATCH] Update robots.txt with new URL endpoints; #504 --- robots.txt | 63 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/robots.txt b/robots.txt index aca189f5fe..3cd75447c6 100644 --- a/robots.txt +++ b/robots.txt @@ -1,21 +1,50 @@ User-agent: * -Disallow: /admin/ -Disallow: /accounts/ +Disallow: /500 +Disallow: /accounts +Disallow: /admin +Disallow: /api +Disallow: /comments +Disallow: /contest/*/clone +Disallow: /contest/*/join +Disallow: /contest/*/leave +Disallow: /contest/*/moss +Disallow: /contest/*/participation +Disallow: /contest/*/participations +Disallow: /contest/*/rank +Disallow: /contests/tag +Disallow: /contest/*/stats +Disallow: /contest/*/submissions +Disallow: /data +Disallow: /edit/profile/$ +Disallow: /feed +Disallow: /judge-select2 +Disallow: /mailgun/mail_activate/$ +Disallow: /organization/*/edit +Disallow: /organization/*/join +Disallow: /organization/*/kick +Disallow: /organization/*/leave +Disallow: /organization/*/request +Disallow: /organization/*/requests +Disallow: /problem/*/clone +Disallow: /problem/*/data +Disallow: /problem/*/manage/submission Disallow: /problem/*/pdf +Disallow: /problem/*/rank Disallow: /problem/*/raw +Disallow: /problem/*/resubmit +Disallow: /problems/random/$ +Disallow: /problem/*/submissions Disallow: /problem/*/submit -Disallow: /problem/*/resubmit/ -Disallow: /problem/*/submissions/ -Disallow: /user/*/submissions/ -Disallow: /submission/ -Disallow: /submissions/ -Disallow: /single_submission -Disallow: /edit/profile/ -Disallow: /src/ -Disallow: /comments/ -Disallow: /api/ -Disallow: /contest/*/submissions/*/*/ -Disallow: /contest/*/join -Disallow: /contest/*/leave -Disallow: /contests/*/*/ -Disallow: /tickets/* +Disallow: /problem/*/test_data +Disallow: /problem/*/tickets +Disallow: /src +Disallow: /stats +Disallow: /submission +Disallow: /submissions +Disallow: /tasks +Disallow: /ticket +Disallow: /tickets +Disallow: /user/$ +Disallow: /users/find/$ +Disallow: /user/*/submissions +Disallow: /widgets