From adfa8b1aa4b3e066102710a005806b8d130552a8 Mon Sep 17 00:00:00 2001
From: Oliver Beckstein <orbeckst@gmail.com>
Date: Fri, 10 Jul 2020 16:15:07 -0700
Subject: [PATCH 1/3] update mdanalysis.json

- use sitemapindex.xml instead of start_urls
  - www.mdanalysis.org
  - docs.mdanalysis.org
  - userguide.mdanalysis.org
- stop indexing old docs (www.mdanalysis.org/docs and www.mdanalysis.org/mdanalysis)
- stop indexing generated docs for the three domains (not sure if we can use regular expressions for subdomains - to be safe, I repeated them)
- completes https://github.com/MDAnalysis/MDAnalysis.github.io/issues/79
---
 configs/mdanalysis.json | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/configs/mdanalysis.json b/configs/mdanalysis.json
index 44d4c34d1..73685a139 100644
--- a/configs/mdanalysis.json
+++ b/configs/mdanalysis.json
@@ -1,17 +1,33 @@
 {
   "index_name": "mdanalysis",
   "sitemap_urls": [
-    "https://www.mdanalysis.org/sitemap.xml"
+    "https://www.mdanalysis.org/sitemapindex.xml"
   ],
   "start_urls": [
-    "https://www.mdanalysis.org"
   ],
   "stop_urls": [
     "https://www.mdanalysis.org/.*?//.*?",
+    "https://www.mdanalysis.org/blog/page.*",
     "https://www.mdanalysis.org/mdanalysis",
-    "https://www.mdanalysis.org/docs/_modules/",
-    "https://www.mdanalysis.org/docs/_sources",
-    "https://www.mdanalysis.org/blog/page.*"
+    "https://www.mdanalysis.org/docs",
+    {
+      "url": "https://www.mdanalysis.org/.*?/(?P<generated>.*?)/",
+      "variables": {
+        "generated": ["_modules", "_sources"]
+      }
+    },    
+    {
+      "url": "https://docs.mdanalysis.org/.*?/(?P<generated>.*?)/",
+      "variables": {
+        "generated": ["_modules", "_sources"]
+      }
+    },
+    {
+      "url": "https://userguide.mdanalysis.org/.*?/(?P<generated>.*?)/",
+      "variables": {
+        "generated": ["_modules", "_sources"]
+      }
+    },      
   ],
   "selectors": {
     "lvl0": {
@@ -29,4 +45,4 @@
     "569445928"
   ],
   "nb_hits": 1700
-}
\ No newline at end of file
+}

From dd48ef6324cfbfbcf986046292b1336f34bd1e8d Mon Sep 17 00:00:00 2001
From: Oliver Beckstein <orbeckst@gmail.com>
Date: Fri, 10 Jul 2020 16:26:27 -0700
Subject: [PATCH 2/3] fixed json

---
 configs/mdanalysis.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/mdanalysis.json b/configs/mdanalysis.json
index 73685a139..4c5cf0496 100644
--- a/configs/mdanalysis.json
+++ b/configs/mdanalysis.json
@@ -27,7 +27,7 @@
       "variables": {
         "generated": ["_modules", "_sources"]
       }
-    },      
+    }      
   ],
   "selectors": {
     "lvl0": {

From 2405ccc99261ec0aa8a01b357a493e3d359bd747 Mon Sep 17 00:00:00 2001
From: Oliver Beckstein <orbeckst@gmail.com>
Date: Wed, 15 Jul 2020 00:52:31 -0700
Subject: [PATCH 3/3] Update configs/mdanalysis.json

Fix stop_urls

Co-authored-by: Sylvain Pace <sylvain.pace@algolia.com>
---
 configs/mdanalysis.json | 19 +------------------
 1 file changed, 1 insertion(+), 18 deletions(-)

diff --git a/configs/mdanalysis.json b/configs/mdanalysis.json
index 4c5cf0496..d5f657e1c 100644
--- a/configs/mdanalysis.json
+++ b/configs/mdanalysis.json
@@ -10,24 +10,7 @@
     "https://www.mdanalysis.org/blog/page.*",
     "https://www.mdanalysis.org/mdanalysis",
     "https://www.mdanalysis.org/docs",
-    {
-      "url": "https://www.mdanalysis.org/.*?/(?P<generated>.*?)/",
-      "variables": {
-        "generated": ["_modules", "_sources"]
-      }
-    },    
-    {
-      "url": "https://docs.mdanalysis.org/.*?/(?P<generated>.*?)/",
-      "variables": {
-        "generated": ["_modules", "_sources"]
-      }
-    },
-    {
-      "url": "https://userguide.mdanalysis.org/.*?/(?P<generated>.*?)/",
-      "variables": {
-        "generated": ["_modules", "_sources"]
-      }
-    }      
+    "\\/_"
   ],
   "selectors": {
     "lvl0": {