update robots.txt

ncbo · Nov 21, 2024 · 8fcdded · 8fcdded
1 parent 45b564f
commit 8fcdded
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 2 deletions.
diff --git a/config/robots/appliance.txt b/config/robots/appliance.txt
@@ -1,9 +1,20 @@
 User-agent: *
+# This will match '/mappings', as well - due to path value wildcard [
+# https://developers.google.com/search/reference/robots_txt#url-matching-based-on-path-values ].
 Disallow: /*mappings
+Disallow: /projects
+Disallow: /projects/
 Disallow: /login
 Disallow: /virtual
 Disallow: /visualize
-Crawl-delay: 2
+Disallow: /feedback
+Disallow: /ajax
+Disallow: /javascripts
+Disallow: /widgets
+Crawl-delay: 5
+
+User-agent: Amazonbot
+Crawl-delay: 10
 
 User-agent: SemrushBot
 Disallow: /
@@ -17,4 +28,16 @@ Disallow: /
 User-agent: MJ12bot
 Disallow: /
 
+User-agent: AhrefsBot
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /
+
+User-agent: BLEXBot
+Disallow: /
+
+User agent: MTRobot
+Disallow: /
+
 
diff --git a/config/robots/production.txt b/config/robots/production.txt
@@ -7,10 +7,15 @@ Disallow: /projects/
 Disallow: /login
 Disallow: /virtual
 Disallow: /visualize
-Disallow: /resource_index
 Disallow: /feedback
+Disallow: /ajax
+Disallow: /javascripts
+Disallow: /widgets
 Crawl-delay: 5
 
+User-agent: Amazonbot
+Crawl-delay: 10
+
 User-agent: SemrushBot
 Disallow: /
 
@@ -23,5 +28,16 @@ Disallow: /
 User-agent: MJ12bot
 Disallow: /
 
+User-agent: AhrefsBot
+Disallow: /
+
+User-agent: DataForSeoBot
+Disallow: /
+
+User-agent: BLEXBot
+Disallow: /
+
+User agent: MTRobot
+Disallow: /