Fix issue 2045 (#3522)

dnnsoftware · Jan 27, 2020 · ba6b3bf · ba6b3bf
1 parent 0cc3479
commit ba6b3bf
Showing 1 changed file with 1 addition and 130 deletions.
diff --git a/DNN Platform/Website/Robots.txt b/DNN Platform/Website/Robots.txt
@@ -4,104 +4,7 @@
 #\-----------------------------------------------/
 #Sitemap: http://www.DomainNamehere.com/sitemap.aspx
 
-
 User-agent: *
-Disallow: /admin/
-Disallow: /App_Browsers/
-Disallow: /App_Code/
-Disallow: /App_Data/
-Disallow: /App_GlobalResources/
-Disallow: /bin/
-Disallow: /Components/
-Disallow: /Config/
-Disallow: /contest/
-Disallow: /controls/
-Disallow: /DesktopModules/
-Disallow: /Documentation/
-Disallow: /HttpModules/
-Disallow: /images/
-Disallow: /Install/
-Disallow: /js/
-Disallow: /Portals/
-Disallow: /Providers/
-Disallow: /Resources/ContentRotator/
-Disallow: /Resources/ControlPanel/
-Disallow: /Resources/Dashboard/
-Disallow: /Resources/FeedBrowser/
-Disallow: /Resources/OpenForceAd/
-Disallow: /Resources/Search/
-Disallow: /Resources/Shared/
-Disallow: /Resources/SkinWidgets/
-Disallow: /Resources/TabStrip/
-Disallow: /Resources/Widgets/
-Disallow: /Activity-Feed/userId/	# Do not index user profiles
-
-User-agent: msnbot
-Disallow: /admin/
-Disallow: /App_Browsers/
-Disallow: /App_Code/
-Disallow: /App_Data/
-Disallow: /App_GlobalResources/
-Disallow: /bin/
-Disallow: /Components/
-Disallow: /Config/
-Disallow: /contest/
-Disallow: /controls/
-Disallow: /DesktopModules/
-Disallow: /Documentation/
-Disallow: /HttpModules/
-Disallow: /images/
-Disallow: /Install/
-Disallow: /js/
-Disallow: /Portals/
-Disallow: /Providers/
-Disallow: /Resources/ContentRotator/
-Disallow: /Resources/ControlPanel/
-Disallow: /Resources/Dashboard/
-Disallow: /Resources/FeedBrowser/
-Disallow: /Resources/OpenForceAd/
-Disallow: /Resources/Search/
-Disallow: /Resources/Shared/
-Disallow: /Resources/SkinWidgets/
-Disallow: /Resources/TabStrip/
-Disallow: /Resources/Widgets/
-Disallow: /Activity-Feed/userId/	# Do not index user profiles
-Crawl-delay: 5
-
-User-agent: Slurp
-Disallow: /*/ctl/		# Slurp permits *
-Disallow: /admin/
-Disallow: /App_Browsers/
-Disallow: /App_Code/
-Disallow: /App_Data/
-Disallow: /App_GlobalResources/
-Disallow: /bin/
-Disallow: /Components/
-Disallow: /Config/
-Disallow: /contest/
-Disallow: /controls/
-Disallow: /DesktopModules/
-Disallow: /Documentation/
-Disallow: /HttpModules/
-Disallow: /images/
-Disallow: /Install/
-Disallow: /js/
-Disallow: /Portals/
-Disallow: /Providers/
-Disallow: /Resources/ContentRotator/
-Disallow: /Resources/ControlPanel/
-Disallow: /Resources/Dashboard/
-Disallow: /Resources/FeedBrowser/
-Disallow: /Resources/OpenForceAd/
-Disallow: /Resources/Search/
-Disallow: /Resources/Shared/
-Disallow: /Resources/SkinWidgets/
-Disallow: /Resources/TabStrip/
-Disallow: /Resources/Widgets/
-Disallow: /Activity-Feed/userId/	# Do not index user profiles
-Crawl-delay: 5
-
-User-agent: Googlebot
 Disallow: /*/ctl/		# Googlebot permits *
 Disallow: /admin/
 Disallow: /App_Browsers/
@@ -119,36 +22,4 @@ Disallow: /Install/
 Disallow: /Providers/
 Disallow: /Activity-Feed/userId/	# Do not index user profiles
 
-User-agent: Yahoo Pipes 1.0
-Disallow: /admin/
-Disallow: /App_Browsers/
-Disallow: /App_Code/
-Disallow: /App_Data/
-Disallow: /App_GlobalResources/
-Disallow: /bin/
-Disallow: /Components/
-Disallow: /Config/
-Disallow: /contest/
-Disallow: /controls/
-Disallow: /DesktopModules/
-Disallow: /Documentation/
-Disallow: /HttpModules/
-Disallow: /images/
-Disallow: /Install/
-Disallow: /js/
-Disallow: /Portals/
-Disallow: /Providers/
-Disallow: /Resources/ContentRotator/
-Disallow: /Resources/ControlPanel/
-Disallow: /Resources/Dashboard/
-Disallow: /Resources/FeedBrowser/
-Disallow: /Resources/OpenForceAd/
-Disallow: /Resources/Search/
-Disallow: /Resources/Shared/
-Disallow: /Resources/SkinWidgets/
-Disallow: /Resources/TabStrip/
-Disallow: /Resources/Widgets/
-Disallow: /Activity-Feed/userId/	# Do not index user profiles
-
-
-# End of robots.txt file
+# End of robots.txt file