[DOCS] Add proper robots.txt and sitemap.xml to index only latest and…

… stable (#1752)

Solution proposed here:
https://stackoverflow.com/questions/63542354/readthedocs-robots-txt-and-sitemap-xml

---------

Co-authored-by: Jay Chia <[email protected]@users.noreply.github.com>

docs/source/_html/robots.txt

@@ -0,0 +1,16 @@
+User-agent: *
+
+# Allow crawling of all paths
+Allow: /
+
+# Override to disallow crawling of documentation
+Disallow: /projects/docs/
+
+# Override again to allow crawling of /latest documentation
+Allow: /projects/docs/en/latest/
+
+# Override again to allow crawling of /stable documentation
+Allow: /projects/docs/en/stable/
+
+# Custom sitemap
+Sitemap: https://www.getdaft.io/sitemap-override.xml

docs/source/_html/sitemap-override.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml">
+    <url>
+        <loc>https://www.getdaft.io/projects/docs/en/latest/</loc>
+        <changefreq>daily</changefreq>
+        <priority>1</priority>
+    </url>
+    <url>
+        <loc>https://www.getdaft.io/projects/docs/en/stable/</loc>
+        <changefreq>weekly</changefreq>
+        <priority>0.9</priority>
+    </url>
+    <url>
+        <loc>https://www.getdaft.io/</loc>
+        <changefreq>monthly</changefreq>
+        <priority>0.5</priority>
+    </url>
+</urlset>

docs/source/conf.py

@@ -82,6 +82,9 @@
     "learn/10-min": "../10-min.html",
 }
 
+# Adds /_html folder which serves robots.txt and the sitemap
+html_extra_path = ["_html"]
+
 # Resolving code links to github
 # Adapted from: https://github.com/aaugustin/websockets/blob/778a1ca6936ac67e7a3fe1bbe585db2eafeaa515/docs/conf.py#L100-L134