From 03f6dac818cb3288f17787594c55dedf0ce187b6 Mon Sep 17 00:00:00 2001
From: Jay Chia <17691182+jaychia@users.noreply.github.com>
Date: Wed, 20 Dec 2023 16:11:26 +0800
Subject: [PATCH] [DOCS] Add proper robots.txt and sitemap.xml to index only
 latest and stable (#1752)

Solution proposed here:
https://stackoverflow.com/questions/63542354/readthedocs-robots-txt-and-sitemap-xml

---------

Co-authored-by: Jay Chia <jaychia94@gmail.com@users.noreply.github.com>
---
 docs/source/_html/robots.txt           | 16 ++++++++++++++++
 docs/source/_html/sitemap-override.xml | 19 +++++++++++++++++++
 docs/source/conf.py                    |  3 +++
 3 files changed, 38 insertions(+)
 create mode 100644 docs/source/_html/robots.txt
 create mode 100644 docs/source/_html/sitemap-override.xml

diff --git a/docs/source/_html/robots.txt b/docs/source/_html/robots.txt
new file mode 100644
index 0000000000..f5f081a8eb
--- /dev/null
+++ b/docs/source/_html/robots.txt
@@ -0,0 +1,16 @@
+User-agent: *
+
+# Allow crawling of all paths
+Allow: /
+
+# Override to disallow crawling of documentation
+Disallow: /projects/docs/
+
+# Override again to allow crawling of /latest documentation
+Allow: /projects/docs/en/latest/
+
+# Override again to allow crawling of /stable documentation
+Allow: /projects/docs/en/stable/
+
+# Custom sitemap
+Sitemap: https://www.getdaft.io/sitemap-override.xml
diff --git a/docs/source/_html/sitemap-override.xml b/docs/source/_html/sitemap-override.xml
new file mode 100644
index 0000000000..0d27e95390
--- /dev/null
+++ b/docs/source/_html/sitemap-override.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml">
+    <url>
+        <loc>https://www.getdaft.io/projects/docs/en/latest/</loc>
+        <changefreq>daily</changefreq>
+        <priority>1</priority>
+    </url>
+    <url>
+        <loc>https://www.getdaft.io/projects/docs/en/stable/</loc>
+        <changefreq>weekly</changefreq>
+        <priority>0.9</priority>
+    </url>
+    <url>
+        <loc>https://www.getdaft.io/</loc>
+        <changefreq>monthly</changefreq>
+        <priority>0.5</priority>
+    </url>
+</urlset>
diff --git a/docs/source/conf.py b/docs/source/conf.py
index b186fcb077..2a14d45aa0 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -82,6 +82,9 @@
     "learn/10-min": "../10-min.html",
 }
 
+# Adds /_html folder which serves robots.txt and the sitemap
+html_extra_path = ["_html"]
+
 # Resolving code links to github
 # Adapted from: https://github.com/aaugustin/websockets/blob/778a1ca6936ac67e7a3fe1bbe585db2eafeaa515/docs/conf.py#L100-L134