From 4462606df4a9a9d482aa4bf8973500586d04d6c7 Mon Sep 17 00:00:00 2001
From: Adam Collard <sparkiegeek@users.noreply.github.com>
Date: Fri, 21 Apr 2023 15:58:55 +0100
Subject: [PATCH] Upgrade Discourse docs to v5, use revised index topic (#772)

* feat: port to discourse docs v5, extract init_docs, init_tutorials

---------

Co-authored-by: Francisco Jimenez Cabrera <francisco.jimenez.cabrera@canonical.com>
Co-authored-by: Peter Makowski <me+git@petermakowski.io>
---
 konf/site.yaml                 |  10 +
 requirements.txt               |   2 +-
 templates/docs/document.html   | 282 ++++++++++++++++++++---
 templates/tutorials/index.html | 112 +--------
 tests/test_doc_parser.py       | 154 -------------
 webapp/app.py                  | 143 +++---------
 webapp/doc_parser.py           | 403 ---------------------------------
 webapp/openapi_parser.py       |   3 +-
 webapp/views/__init__.py       |   0
 webapp/views/docs.py           |  44 ++++
 webapp/views/tutorials.py      |  70 ++++++
 11 files changed, 419 insertions(+), 804 deletions(-)
 delete mode 100644 tests/test_doc_parser.py
 delete mode 100644 webapp/doc_parser.py
 create mode 100644 webapp/views/__init__.py
 create mode 100644 webapp/views/docs.py
 create mode 100644 webapp/views/tutorials.py

diff --git a/konf/site.yaml b/konf/site.yaml
index 9dd928ca..4b2ef6e9 100644
--- a/konf/site.yaml
+++ b/konf/site.yaml
@@ -11,6 +11,16 @@ env:
       key: google-custom-search-key
       name: google-api
 
+  - name: DISCOURSE_API_KEY
+    secretKeyRef:
+      key: maas-api-key
+      name: discourse-api
+
+  - name: DISCOURSE_API_USERNAME
+    secretKeyRef:
+      key: maas-api-username
+      name: discourse-api
+
 extraHosts:
   - domain: docs.maas.io
   - domain: maas.ubuntu.com
diff --git a/requirements.txt b/requirements.txt
index 839b6554..b7d2c917 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 canonicalwebteam.flask_base==1.1.0
 canonicalwebteam.blog==6.4.0
-canonicalwebteam.discourse-docs==1.0.1
+canonicalwebteam.discourse==5.4.1
 canonicalwebteam.http==1.0.3
 canonicalwebteam.templatefinder==1.0.0
 canonicalwebteam.search==1.3.0
diff --git a/templates/docs/document.html b/templates/docs/document.html
index a9ca4491..c42f4285 100644
--- a/templates/docs/document.html
+++ b/templates/docs/document.html
@@ -8,6 +8,40 @@
 {% block og_title %}{{ document.title }}{% endblock %}
 {% block og_description %}{% endblock %}
 
+{% macro create_navigation(nav_items, expandable=False, expanded=False) %}
+  <ul class="p-side-navigation__list">
+    {% for element in nav_items %}
+    <li class="p-side-navigation__item">
+      {% if element.navlink_href %}
+      <a
+        class="p-side-navigation__link {% if expandable and element.children %}is-expandable{% endif %}"
+        href="{{ element.navlink_href }}"
+        {% if expandable and element.children %}aria-expanded={% if expanded or element.is_active %}"true"{% else %}"false"{% endif %}{% endif %}
+        {% if element.is_active and not element.navlink_fragment %}aria-current="page"{% endif %}
+      >{{ element.navlink_text }}</a>
+      {% else %}
+        <strong
+          class="p-side-navigation__text {% if expandable and element.children %}is-expandable{% endif %}"
+          {% if expandable and element.children %}aria-expanded={% if expanded %}"true"{% else %}"false"{% endif %}{% endif %}
+          {% if element.is_active %}aria-current="page"{% endif %}
+        >{{ element.navlink_text }}</strong>
+      {% endif %}
+
+      {% if expandable %}
+        {% if element.children %}
+            <button class="p-side-navigation__expand" aria-expanded={% if element.is_active or element.has_active_child %}"true"{% else %}"false"{% endif %} aria-label="show submenu for {{ element.navlink_text }}"></button>
+        {% endif %}
+        {{ create_navigation(element.children, expandable, element.is_active or element.has_active_child) }}
+      {% else %}
+        {% if element.children %}
+          {{ create_navigation(element.children, expandable) }}
+        {% endif %}
+      {% endif %}
+    </li>
+    {% endfor %}
+  </ul>
+{% endmacro %}
+
 {% block content %}
   <section id="search-docs" class="p-strip--light is-shallow">
     <div class="row">
@@ -18,68 +52,252 @@
       </form>
     </div>
   </section>
+
   <div class="l-docs">
-    <aside class="l-docs-sidebar" id="navigation">
-      <i class="p-sidenav__toggle p-icon--menu u-hide u-show--small"></i>
-      <nav class="p-sidenav__body u-hide--small">
-        {{ navigation | safe }}
-      </nav>
+    <aside class="col-3 l-docs-sidebar">
+      {% if versions | length > 1 %}
+      <label for="version-select" class="u-hide">Version</label>
+      <select name="version-select" id="version-select" onChange="window.location.href=this.value">
+      {% for version in versions %}
+        {% set active = docs_version == version['path'] %}
+        <option value="{{ version_paths[version['path']] }}"{% if active %} selected{% endif %}>{{ version['version'] }}</option>
+      {% endfor %}
+      <select>
+      {% endif %}
+
+      <div class="row">
+	<div class="p-strip is-shallow">
+	  <nav data-js="navigation" class="p-side-navigation" id="{{ navigation['path'] or 'default' }}">
+	  <a href="#{{ navigation['path'] or 'default' }}" class="p-side-navigation__toggle js-drawer-toggle" aria-controls="{{ navigation['path'] or 'default' }}">
+	    Toggle side navigation
+	  </a>
+	  <div class="p-side-navigation__overlay js-drawer-toggle" aria-controls="{{ navigation['path'] or 'default' }}"></div>
+	  <div class="p-side-navigation__drawer">
+	    <div class="p-side-navigation__drawer-header">
+	      <a href="#" class="p-side-navigation__toggle--in-drawer js-drawer-toggle" aria-controls="{{ navigation['path'] or 'default' }}">
+		Toggle side navigation
+	      </a>
+	    </div>
+	    {% for nav_group in navigation.nav_items %}
+	    {% if not nav_group.hidden %}
+	      {% if nav_group.navlink_text %}
+		{% if nav_group.navlink_href %}
+		<h3 class="p-side-navigation__heading--linked">
+		  <a class="p-side-navigation__link" href="{{ nav_group.navlink_href }}" {% if nav_group.is_active %}aria-current="page"{% endif %}>
+		    {{ nav_group.navlink_text }}
+		  </a>
+		</h3>
+		{% else %}
+		  <h3 class="p-side-navigation__heading">{{ nav_group.navlink_text }}</h3>
+		{% endif %}
+	      {% endif %}
+	      {#
+		Use `create_navigation(nav_group.children)` for a default, fully expanded navigation.
+		Use `create_navigation(nav_group.children, expandable=True)` for the nested nav levels to expand only when parent page is active.
+	      #}
+	      {{ create_navigation(nav_group.children, expandable=True) }}
+	    {% endif %}
+	  {% endfor %}
+	  </div>
+	</nav>
+      </div>
+    </div>
     </aside>
-    <main class="l-docs-content" id="main-content">
+    <main class="col-9 l-docs-content" id="content-section">
       <div class="p-strip--light is-shallow l-docs-title">
         <div class="l-docs-row row">
-          <h1 class="u-no-margin--bottom l-docs-title__heading">{{ document.title }}</h1>
+          <h1 class="l-docs-title__heading u-no-margin--bottom">{{ document.title }}</h1>
         </div>
       </div>
-
       <div class="p-strip is-shallow">
         <div class="l-docs-row row">
           {{ document.body_html | safe }}
         </div>
       </div>
-
       <div class="p-strip is-shallow u-no-padding--bottom">
         <div class="l-docs-row row">
           <hr />
-          <i>Last updated {{ document.updated }}.</i>
+          <p><i>Last updated {{ document.updated }}.</i></p>
         </div>
       </div>
       <div class="p-strip is-shallow u-no-padding--bottom u-sticky--bottom">
         <div class="l-docs-row row">
           <div class="p-notification--information">
-            <p class="p-notification__content">
-              <a href="{{ forum_url }}{{ document.topic_path }}">Help improve this document in the forum</a>.
-            </p>
+            <div class="p-notification__content">
+              <p class="p-notification__message">
+                <a href="{{ forum_url }}{{ document.topic_path }}">Help improve this document in the forum</a>.
+              </p>
+            </div>
           </div>
         </div>
       </div>
     </main>
   </div>
+{% endblock %}
+{% block js_extra %}
   <script src="/static/js/highlight.pack.js"></script>
   <script>
-    // Set up highlightjs
-    hljs.initHighlightingOnLoad();
-
-    document.addEventListener("DOMContentLoaded", function(event) {
-      // Toggle mobile sidebar nav
-      var toggle = document.querySelector('.p-sidenav__toggle');
-      var sidebarContent = document.querySelector('.p-sidenav__body');
-      toggle.addEventListener('click', function(e) {
-        toggle.classList.toggle('p-icon--menu');
-        toggle.classList.toggle('p-icon--close');
-        sidebarContent.classList.toggle('u-hide--small');
-      });
+  window.addEventListener("DOMContentLoaded", function() {
+       // Based on Vanilla side navigation example
+// Should be moved out from the template as part of a given project JS 
+  /**
+    Toggles the expanded/collapsed classed on side navigation element.
+    @param {HTMLElement} sideNavigation The side navigation element.
+    @param {Boolean} show Whether to show or hide the drawer.
+  */
+  function toggleDrawer(sideNavigation, show) {
+    const toggleButtonOutsideDrawer = sideNavigation.querySelector(
+      ".p-side-navigation__toggle"
+    );
+    const toggleButtonInsideDrawer = sideNavigation.querySelector(
+      ".p-side-navigation__toggle--in-drawer"
+    );
+
+    if (sideNavigation) {
+      if (show) {
+        sideNavigation.classList.remove("is-drawer-collapsed");
+        sideNavigation.classList.add("is-drawer-expanded");
+
+        toggleButtonInsideDrawer.focus();
+        toggleButtonOutsideDrawer.setAttribute("aria-expanded", true);
+        toggleButtonInsideDrawer.setAttribute("aria-expanded", true);
+      } else {
+        sideNavigation.classList.remove("is-drawer-expanded");
+        sideNavigation.classList.add("is-drawer-collapsed");
+
+        toggleButtonOutsideDrawer.focus();
+        toggleButtonOutsideDrawer.setAttribute("aria-expanded", false);
+        toggleButtonInsideDrawer.setAttribute("aria-expanded", false);
+      }
+    }
+  }
+
+  // throttle util (for window resize event)
+  var throttle = function (fn, delay) {
+    var timer = null;
+    return function () {
+      var context = this,
+        args = arguments;
+      clearTimeout(timer);
+      timer = setTimeout(function () {
+        fn.apply(context, args);
+      }, delay);
+    };
+  };
+
+  /**
+    Attaches event listeners for the side navigation toggles
+    @param {HTMLElement} sideNavigation The side navigation element.
+  */
+  function setupSideNavigation(sideNavigation) {
+    var toggles = [].slice.call(
+      sideNavigation.querySelectorAll(".js-drawer-toggle")
+    );
+    var drawerEl = sideNavigation.querySelector(".p-side-navigation__drawer");
+
+    // hide navigation drawer on small screens
+    sideNavigation.classList.add("is-drawer-hidden");
+
+    // setup drawer element
+    drawerEl.addEventListener("animationend", () => {
+      if (!sideNavigation.classList.contains("is-drawer-expanded")) {
+        sideNavigation.classList.add("is-drawer-hidden");
+      }
     });
 
-    // Add active links to active nav item and scroll to section
-    var sidebar = document.querySelector(".l-docs-sidebar");
-    document.querySelectorAll('.p-sidenav__body ul a').forEach(function(anchor) {
-      if (anchor.pathname === location.pathname) {
-        anchor.classList.add('is-active');
-        var section = anchor.closest("ul");
-        sidebar.scrollTo(0, section.offsetTop - 40);
+    window.addEventListener("keydown", (e) => {
+      if (e.key === "Escape") {
+        toggleDrawer(sideNavigation, false);
       }
     });
+
+    // setup toggle buttons
+    toggles.forEach(function (toggle) {
+      toggle.addEventListener("click", function (event) {
+        event.preventDefault();
+
+        if (sideNavigation) {
+          sideNavigation.classList.remove("is-drawer-hidden");
+          toggleDrawer(
+            sideNavigation,
+            !sideNavigation.classList.contains("is-drawer-expanded")
+          );
+        }
+      });
+    });
+
+    // hide side navigation drawer when screen is resized
+    window.addEventListener(
+      "resize",
+      throttle(function () {
+        toggles.forEach((toggle) => {
+          return toggle.setAttribute("aria-expanded", false);
+        });
+        // remove expanded/collapsed class names to avoid unexpected animations
+        sideNavigation.classList.remove("is-drawer-expanded");
+        sideNavigation.classList.remove("is-drawer-collapsed");
+        sideNavigation.classList.add("is-drawer-hidden");
+      }, 10)
+    );
+  }
+
+  /**
+    Attaches event listeners for all the side navigations in the document.
+    @param {String} sideNavigationSelector The CSS selector matching side navigation elements.
+  */
+  function setupSideNavigations(sideNavigationSelector) {
+    // Setup all side navigations on the page.
+    var sideNavigations = [].slice.call(
+      document.querySelectorAll(sideNavigationSelector)
+    );
+
+    sideNavigations.forEach(setupSideNavigation);
+  }
+
+  setupSideNavigations('.p-side-navigation, [class*="p-side-navigation--"]');
+
+  // Setup expandable side navigation
+
+  var expandToggles = document.querySelectorAll(".p-side-navigation__expand");
+  var navigationLinks = document.querySelectorAll(".p-side-navigation__link");
+
+  // setup default values of aria-expanded for the toggle button, list title and list itself
+  const setup = (toggle) => {
+    const isExpanded = toggle.getAttribute("aria-expanded") === "true";
+    if (!isExpanded) {
+      toggle.setAttribute("aria-expanded", isExpanded);
+    }
+    const item = toggle.closest(".p-side-navigation__item");
+    const link = item.querySelector(".p-side-navigation__link");
+    const nestedList = item.querySelector(".p-side-navigation__list");
+    if (!link?.hasAttribute("aria-expanded")) {
+      link.setAttribute("aria-expanded", isExpanded);
+    }
+    if (!nestedList?.hasAttribute("aria-expanded")) {
+      nestedList.setAttribute("aria-expanded", isExpanded);
+    }
+  };
+
+  const handleToggle = (e) => {
+    const item = e.currentTarget.closest(".p-side-navigation__item");
+    const button = item.querySelector(".p-side-navigation__expand");
+    const link = item.querySelector(".p-side-navigation__link");
+    const nestedList = item.querySelector(".p-side-navigation__list");
+    [button, link, nestedList].forEach((el) =>
+      el.setAttribute(
+        "aria-expanded",
+        el.getAttribute("aria-expanded") === "true" ? "false" : "true"
+      )
+    );
+  };
+
+  expandToggles.forEach((toggle) => {
+    setup(toggle);
+    toggle.addEventListener("click", (e) => {
+      handleToggle(e);
+    });
+  });
+  });
   </script>
   <script src="/static/js/modules/discourse-rad-parser/discourse-rad-parser.js"></script>
   <script>
diff --git a/templates/tutorials/index.html b/templates/tutorials/index.html
index 895b1f1a..775fe38a 100644
--- a/templates/tutorials/index.html
+++ b/templates/tutorials/index.html
@@ -12,18 +12,18 @@ <h1 class="p-heading--2 u-no-margin--bottom">Tutorials</h1>
 
 <section class="p-strip is-shallow">
   <div class="row u-equal-height">
-    {% for item in metadata %}
-      {% if loop.index <= posts_per_page * page and loop.index >= posts_per_page * page - posts_per_page + 1 %}
-        <div class="col-4 col-medium-3 p-card">
-          <div class="p-card__content">
-            <h3 class="p-heading--4">
-              {{ item.topic | safe }}
-            </h3>
-            <p>{{ item.summary | safe }}</p>
-          </div>
-          <p>Difficulty: <span class="p-tutorials-meter p-tutorials-meter--{{ item.difficulty }}">{{ item.difficulty }} out of 5</span></p>
-        </div>
-      {% endif %}
+    {% for item in tutorials %}
+      <div class="col-4 col-medium-3 p-card">
+	<div class="p-card__content">
+	  <h3 class="p-heading--4">
+	    <a href="{{ item.link }}">
+	      {{ item.title | safe }}
+	    </a>
+	  </h3>
+	  <p>{{ item.summary | safe }}</p>
+	</div>
+	<p>Difficulty: <span class="p-tutorials-meter p-tutorials-meter--{{ item.difficulty }}">{{ item.difficulty }} out of 5</span></p>
+      </div>
     {% endfor %}
   </div>
 </section>
@@ -37,92 +37,4 @@ <h3 class="p-heading--4">
   </div>
 </section>
 
-
-{% if total_pages > 1 %}
-<section class="p-strip">
-  <div class="u-fixed-width">
-    <ol class="p-pagination u-align--center">
-      {% set current_page = page %}
-      {% if current_page > 1 %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link--previous" href="/tutorials?page={{ current_page - 1 }}" title="Previous page">
-            <i class="p-icon--chevron-down">Previous page</i>
-          </a>
-        </li>
-      {% else %}
-        <li class="p-pagination__item">
-          <span class="p-pagination__link--previous is-disabled"><i class="p-icon--chevron-down">Previous page</i></span>
-        </li>
-      {% endif %}
-
-      {# always show 5 pages in pagination #}
-      {% if current_page > 4 and current_page == total_pages %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link" href="/tutorials?page={{ current_page - 4 }}">{{ current_page - 4 }}</a>
-        </li>
-      {% endif %}
-
-      {% if current_page > 3 and current_page >= total_pages - 1 %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link" href="/tutorials?page={{ current_page - 3 }}">{{ current_page - 3 }}</a>
-        </li>
-      {% endif %}
-
-      {% if current_page > 2 %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link" href="/tutorials?page={{ current_page - 2 }}">{{ current_page - 2 }}</a>
-        </li>
-      {% endif %}
-
-      {% if current_page > 1 %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link" href="/tutorials?page={{ current_page - 1 }}">{{ current_page - 1 }}</a>
-        </li>
-      {% endif %}
-
-      <!-- current page -->
-      <li class="p-pagination__item">
-        <a class="p-pagination__link is-active" href="/tutorials?page={{ current_page }}">{{ current_page }}</a>
-      </li>
-
-      {% if current_page < total_pages %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link" href="/tutorials?page={{ current_page + 1 }}">{{ current_page + 1 }}</a>
-        </li>
-      {% endif %}
-
-      {% if current_page < total_pages - 1 %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link" href="/tutorials?page={{ current_page + 2 }}">{{ current_page + 2 }}</a>
-        </li>
-      {% endif %}
-
-      {% if current_page < total_pages - 2 and current_page <= 2 %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link" href="/tutorials?page={{ current_page + 3 }}">{{ current_page + 3 }}</a>
-        </li>
-      {% endif %}
-
-      {% if current_page < total_pages - 3 and current_page == 1 %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link" href="/tutorials?page={{ current_page + 4 }}">{{ current_page + 4 }}</a>
-        </li>
-      {% endif %}
-
-      {% if current_page != total_pages %}
-        <li class="p-pagination__item">
-          <a class="p-pagination__link--next" href="/tutorials?page={{ current_page + 1 }}" title="Next page">
-            <i class="p-icon--chevron-down">Next page</i>
-          </a>
-        </li>
-      {% else %}
-        <li class="p-pagination__item">
-          <span class="p-pagination__link--next is-disabled"><i class="p-icon--chevron-down">Next page</i></span>
-        </li>
-      {% endif %}
-    </ol>
-  </div>
-</section>
-{% endif %}
-
 {% endblock %}
diff --git a/tests/test_doc_parser.py b/tests/test_doc_parser.py
deleted file mode 100644
index df26f9ac..00000000
--- a/tests/test_doc_parser.py
+++ /dev/null
@@ -1,154 +0,0 @@
-import json
-import unittest
-import warnings
-
-from bs4 import BeautifulSoup
-import httpretty
-import requests
-
-from canonicalwebteam.discourse_docs import DiscourseAPI
-from webapp.doc_parser import FastDocParser
-
-EXAMPLE_CONTENT = (
-    "<p>Some homepage content</p>"
-    "<h1>Navigation</h1>"
-    "<ul>"
-    '<li><a href="/t/page-a/10">Page A</a></li>'
-    '<li><a href="/t/b-page/12">B page</a></li>'
-    "</ul>"
-    "<h1>URLs</h1>"
-    '<details open="">'
-    "<summary>Mapping table</summary>"
-    '<div class="md-table">'
-    "<table>"
-    "<thead><tr>"
-    "<th>Topic</th><th>Path</th></tr></thead>"
-    "<tbody><tr>"
-    '<td><a href="https://discourse.example.com/t/'
-    'page-a/10">Page A</a></td>'
-    "<td>/a</td>"
-    "</tr><tr>"
-    '<td><a href="https://discourse.example.com/t/'
-    'page-z/26">Page Z</a></td>'
-    "<td>/page-z</td>"
-    "</tr></tbody></table>"
-    "</div></details>"
-    "<h1>Redirects</h1>"
-    '<details open="">'
-    "<summary>Mapping table</summary>"
-    '<div class="md-table">'
-    "<table>"
-    "<thead><tr>"
-    "<th>Topic</th><th>Path</th></tr></thead>"
-    "<tbody>"
-    "<tr><td>/redir-a</td><td>/a</td></tr>"
-    "<tr>"
-    "  <td>/example/page</td>"
-    "  <td>https://example.com/page</td>"
-    "</tr>"
-    "</tr></tbody></table>"
-    "</div></details>"
-)
-
-
-class TestFastDocParser(unittest.TestCase):
-    def setUp(self):
-        # Suppress annoying warnings from HTTPretty
-        # See: https://github.com/gabrielfalcao/HTTPretty/issues/368
-        warnings.filterwarnings(
-            "ignore", category=ResourceWarning, message="unclosed.*"
-        )
-
-        # Enable HTTPretty and set up mock URLs
-        httpretty.enable()
-        self.addCleanup(httpretty.disable)
-        self.addCleanup(httpretty.reset)
-        # Index page with navigation, URL map and redirects
-        httpretty.register_uri(
-            httpretty.GET,
-            "https://discourse.example.com/t/34.json",
-            body=json.dumps(
-                {
-                    "id": 34,
-                    "category_id": 2,
-                    "title": "An index page",
-                    "slug": "an-index-page",
-                    "post_stream": {
-                        "posts": [
-                            {
-                                "id": 3434,
-                                "cooked": EXAMPLE_CONTENT,
-                                "updated_at": "2018-10-02T12:45:44.259Z",
-                            }
-                        ]
-                    },
-                }
-            ),
-            content_type="application/json",
-        )
-
-        discourse_api = DiscourseAPI(
-            base_url="https://discourse.example.com/",
-            session=requests.Session(),
-        )
-
-        self.parser = FastDocParser(
-            api=discourse_api,
-            index_topic_id=34,
-            url_prefix="/",
-        )
-        self.parser.parse()
-
-    def test_index_has_no_nav(self):
-        index = self.parser.index_document
-        soup = BeautifulSoup(index["body_html"], features="lxml")
-
-        # Check body
-        self.assertEqual(soup.p.string, "Some homepage content")
-
-        # Check navigation
-        self.assertIsNone(soup.h1)
-
-        # Check URL map worked
-        self.assertIsNone(soup.details)
-        self.assertNotIn(
-            '<a href="/t/page-a/10">Page A</a>',
-            soup.decode_contents(),
-        )
-
-    def test_nav(self):
-        navigation = self.parser.navigation
-
-        self.assertIn(
-            '<li><a href="/t/b-page/12">B page</a></li>',
-            navigation,
-        )
-
-        self.assertIn('<a href="/a">Page A</a>', navigation)
-
-    def test_redirect_map(self):
-        self.assertEqual(
-            self.parser.redirect_map,
-            {"/redir-a": "/a", "/example/page": "https://example.com/page"},
-        )
-
-        self.assertEqual(self.parser.warnings, [])
-
-    def test_url_map(self):
-        self.assertEqual(
-            self.parser.url_map,
-            {
-                10: "/a",
-                26: "/page-z",
-                34: "/",
-                "/": 34,
-                "/a": 10,
-                "/page-z": 26,
-            },
-        )
-
-        self.assertEqual(self.parser.warnings, [])
-
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/webapp/app.py b/webapp/app.py
index 0ffbc5a8..f3cdb39e 100644
--- a/webapp/app.py
+++ b/webapp/app.py
@@ -1,28 +1,21 @@
 """
 A Flask application for maas.io
 """
+from datetime import timedelta
 
 import flask
-import math
-
-import talisker.requests
-from datetime import timedelta
 from requests_cache import CachedSession
+import talisker.requests
 
-from canonicalwebteam.discourse_docs import (
-    DiscourseDocs,
-    DocParser,
-    DiscourseAPI,
-)
 from canonicalwebteam.flask_base.app import FlaskBase
 from canonicalwebteam.templatefinder import TemplateFinder
-from canonicalwebteam.search import build_search_view
 from canonicalwebteam import image_template
-from webapp.blog.views import init_blog
 
+from webapp.blog.views import init_blog
 from webapp.feeds import get_rss_feed
-from webapp.doc_parser import FastDocParser
 from webapp.openapi_parser import parse_openapi, read_yaml_from_url
+from webapp.views.docs import init_docs
+from webapp.views.tutorials import init_tutorials
 
 from http.client import responses
 
@@ -37,7 +30,6 @@
 app.add_url_rule("/", view_func=template_finder_view)
 app.add_url_rule("/<path:subpath>", view_func=template_finder_view)
 
-session = talisker.requests.get_session()
 docs_session = CachedSession(
     "docs_cache",
     backend="sqlite",
@@ -48,43 +40,10 @@
     match_headers=False,
     stale_if_error=True,
 )
-openapi_session = CachedSession(
-    "openapi_cache",
-    backend="sqlite",
-    cache_control=False,
-    expire_after=timedelta(days=1),
-    allowable_methods=["GET"],
-    allowable_codes=[200, 404, 302, 301],
-    match_headers=False,
-    stale_if_error=True,
-)
+# talisker-ify the cached session, for metrics, logs etc.
+talisker.requests.configure(docs_session)
 
-docs_discourse_api = DiscourseAPI(
-    base_url="https://discourse.maas.io/", session=docs_session
-)
-doc_parser = FastDocParser(
-    api=docs_discourse_api,
-    index_topic_id=25,
-    url_prefix="/docs",
-)
-if app.debug:
-    doc_parser.api.session.adapters["https://"].timeout = 99
-discourse_docs = DiscourseDocs(
-    parser=doc_parser,
-    document_template="docs/document.html",
-    url_prefix="/docs",
-)
-discourse_docs.init_app(app)
-
-
-# Search
-app.add_url_rule(
-    "/docs/search",
-    "docs-search",
-    build_search_view(
-        session=session, site="maas.io/docs", template_path="docs/search.html"
-    ),
-)
+session = talisker.requests.get_session()
 
 
 @app.errorhandler(429)
@@ -107,7 +66,7 @@ def internal_error(error):
 
 @app.context_processor
 def context():
-    return dict(get_rss_feed=get_rss_feed)
+    return {"get_rss_feed": get_rss_feed}
 
 
 @app.context_processor
@@ -115,6 +74,23 @@ def utility_processor():
     return {"image": image_template}
 
 
+openapi_session = CachedSession(
+    "openapi_cache",
+    backend="sqlite",
+    cache_control=False,
+    expire_after=timedelta(days=1),
+    allowable_methods=["GET"],
+    allowable_codes=[200, 404, 302, 301],
+    match_headers=False,
+    stale_if_error=True,
+)
+
+# talisker-ify the cached session, for metrics, logs etc.
+talisker.requests.configure(openapi_session)
+
+docs = init_docs(app, "/docs", session=docs_session)
+
+
 @app.route("/docs/api")
 def api():
     """
@@ -127,76 +103,17 @@ def api():
         "https://raw.githubusercontent.com"
         "/maas/maas-openapi-yaml/main/openapi.yaml"
     )
-    definition = read_yaml_from_url(definition_url, openapi_session)
+    definition = read_yaml_from_url(definition_url, session=openapi_session)
     openapi = parse_openapi(definition)
 
-    doc_parser.parse()
+    docs.parser.parse()
     return flask.render_template(
         "docs/api.html",
-        navigation=doc_parser.navigation,
+        navigation=docs.parser.navigation,
         openapi=openapi,
         responses=responses,
     )
 
 
-url_prefix = "/tutorials"
-tutorials_discourse_api = DiscourseAPI(
-    base_url="https://discourse.maas.io/", session=session
-)
-tutorials_docs_parser = DocParser(
-    api=tutorials_discourse_api,
-    category_id=16,
-    index_topic_id=1289,
-    url_prefix=url_prefix,
-)
-tutorials_docs = DiscourseDocs(
-    parser=tutorials_docs_parser,
-    document_template="/tutorials/tutorial.html",
-    url_prefix=url_prefix,
-    blueprint_name="tutorials",
-)
-
-
-@app.route(url_prefix)
-def index():
-    page = flask.request.args.get("page", default=1, type=int)
-    topic = flask.request.args.get("topic", default=None, type=str)
-    sort = flask.request.args.get("sort", default=None, type=str)
-    posts_per_page = 15
-    tutorials_docs.parser.parse()
-    if not topic:
-        metadata = tutorials_docs.parser.metadata
-    else:
-        metadata = [
-            doc
-            for doc in tutorials_docs.parser.metadata
-            if topic in doc["categories"]
-        ]
-
-    if sort == "difficulty-desc":
-        metadata = sorted(
-            metadata, key=lambda k: k["difficulty"], reverse=True
-        )
-
-    if sort == "difficulty-asc" or not sort:
-        metadata = sorted(
-            metadata, key=lambda k: k["difficulty"], reverse=False
-        )
-
-    total_pages = math.ceil(len(metadata) / posts_per_page)
-
-    return flask.render_template(
-        "tutorials/index.html",
-        navigation=tutorials_docs.parser.navigation,
-        forum_url=tutorials_docs.parser.api.base_url,
-        metadata=metadata,
-        page=page,
-        topic=topic,
-        sort=sort,
-        posts_per_page=posts_per_page,
-        total_pages=total_pages,
-    )
-
-
-tutorials_docs.init_app(app)
 init_blog(app, "/blog")
+init_tutorials(app, "/tutorials", session)
diff --git a/webapp/doc_parser.py b/webapp/doc_parser.py
deleted file mode 100644
index 596f61a1..00000000
--- a/webapp/doc_parser.py
+++ /dev/null
@@ -1,403 +0,0 @@
-# Standard library
-from canonicalwebteam.discourse_docs import DocParser
-from canonicalwebteam.discourse_docs.parsers import TOPIC_URL_MATCH
-
-from functools import cached_property
-import re
-from urllib.parse import urlparse
-
-# Packages
-import dateutil.parser
-import humanize
-import validators
-from bs4 import BeautifulSoup, NavigableString
-from jinja2 import Template
-
-HEADER_REGEX = re.compile("^h[1-6]$")
-
-
-class FastDocParser(DocParser):
-    def parse(self):
-        """
-        Get the index topic and split it into:
-        - navigation
-        - index document content
-        - URL map
-        - redirects map
-        And set those as properties on this object
-        """
-        index_topic = self.api.get_topic(self.index_topic_id)
-        raw_index_soup = BeautifulSoup(
-            index_topic["post_stream"]["posts"][0]["cooked"],
-            features="lxml",
-        )
-
-        # Parse URL & redirects mappings (get warnings)
-        self.url_map, url_warnings = self._parse_url_map(raw_index_soup)
-        self.redirect_map, redirect_warnings = self._parse_redirect_map(
-            raw_index_soup
-        )
-        self.warnings = url_warnings + redirect_warnings
-
-        # Get body and navigation HTML
-        self.index_document = self.parse_topic(
-            index_topic, topic_soup=raw_index_soup
-        )
-        index_soup = self.index_document.pop("body_soup")
-
-        self.index_document["body_html"] = self._get_preamble(
-            index_soup, break_on_title="Navigation"
-        )
-
-        # Parse navigation
-        self.navigation = self._parse_navigation(raw_index_soup)
-
-        self.metadata = None
-        if self.category_id:
-            topics = self.get_all_topics_category()
-            self.metadata = self._parse_metadata(
-                self._replace_links(raw_index_soup, topics)
-            )
-
-    def _parse_redirect_map(self, index_soup):
-        """
-        Given the HTML soup of an index topic
-        extract the redirect mappings from the "Redirects" section.
-
-        The URLs section should contain a table of
-        "Path" to "Location" mappings
-        (extra markup around this table doesn't matter)
-        e.g.:
-
-        <h1>Redirects</h1>
-        <details>
-            <summary>Mapping table</summary>
-            <table>
-            <tr><th>Path</th><th>Location</th></tr>
-            <tr>
-                <td>/my-funky-path</td>
-                <td>/cool-page</td>
-            </tr>
-            <tr>
-                <td>/some/other/path</td>
-                <td>https://example.com/cooler-place</td>
-            </tr>
-            </table>
-        </details>
-
-        This will typically be generated in Discourse from Markdown similar to
-        the following:
-
-        # Redirects
-
-        [details=Mapping table]
-        | Path | Path |
-        | -- | -- |
-        | /my-funky-path | /cool-page |
-        | /some/other/path | https://example.com/cooler-place |
-        """
-
-        redirect_soup = self._get_section(index_soup, "Redirects", "details")
-        redirect_map = {}
-        warnings = []
-
-        if redirect_soup:
-            for row in redirect_soup.select("tr:has(td)"):
-                path_cell = row.select_one("td:first-child")
-                location_cell = row.select_one("td:last-child")
-
-                if not path_cell or not location_cell:
-                    warnings.append(
-                        f"Could not parse redirect map {path_cell}"
-                    )
-                    continue
-
-                path = path_cell.text
-                location = location_cell.text
-
-                if not path.startswith(self.url_prefix):
-                    warnings.append(f"Could not parse redirect map for {path}")
-                    continue
-
-                if not (
-                    location.startswith(self.url_prefix)
-                    or validators.url(location, public=True)
-                ):
-                    warnings.append(
-                        f"Redirect map location {location} is invalid"
-                    )
-                    continue
-
-                if path in self.url_map:
-                    warnings.append(
-                        f"Redirect path {path} clashes with URL map"
-                    )
-                    continue
-
-                redirect_map[path] = location
-
-        return redirect_map, warnings
-
-    def _parse_url_map(self, index_soup):
-        """
-        Given the HTML soup of an index topic
-        extract the URL mappings from the "URLs" section.
-
-        The URLs section should contain a table of
-        "Topic" to "Path" mappings
-        (extra markup around this table doesn't matter)
-        e.g.:
-
-        <h1>URLs</h1>
-        <details>
-            <summary>Mapping table</summary>
-            <table>
-            <tr><th>Topic</th><th>Path</th></tr>
-            <tr>
-                <td><a href="https://forum.example.com/t/page/10">Page</a></td>
-                <td>/cool-page</td>
-            </tr>
-            <tr>
-                <td>
-                  <a href="https://forum.example.com/t/place/11">Place</a>
-                </td>
-                <td>/cool-place</td>
-            </tr>
-            </table>
-        </details>
-
-        This will typically be generated in Discourse from Markdown similar to
-        the following:
-
-        # URLs
-
-        [details=Mapping table]
-        | Topic | Path |
-        | -- | -- |
-        | https://forum.example.com/t/place/11| /cool-page |
-        | https://forum.example.com/t/place/11  | /cool-place |
-
-        """
-        url_soup = self._get_section(index_soup, "URLs", "details")
-
-        url_map = {}
-        warnings = []
-
-        if url_soup:
-            for row in url_soup.tbody("tr"):
-                row_contents = row("td")
-                topic = row_contents[0]
-                path_td = row_contents[-1]
-                topic_a = topic.a
-
-                if not topic_a or not path_td:
-                    warnings.append("Could not parse URL map item {item}")
-                    continue
-
-                topic_url = topic_a.get("href", "")
-                topic_path = urlparse(topic_url).path
-                topic_match = TOPIC_URL_MATCH.match(topic_path)
-
-                pretty_path = path_td.text
-
-                if not topic_match or not pretty_path.startswith(
-                    self.url_prefix
-                ):
-                    warnings.append("Could not parse URL map item {item}")
-                    continue
-
-                topic_id = int(topic_match.groupdict()["topic_id"])
-
-                url_map[pretty_path] = topic_id
-
-        # Add the reverse mappings as well, for efficiency
-        ids_to_paths = dict(reversed(pair) for pair in url_map.items())
-        url_map.update(ids_to_paths)
-
-        # Add the homepage path
-        home_path = self.url_prefix
-        if home_path != "/" and home_path.endswith("/"):
-            home_path = home_path.rstrip("/")
-        url_map[home_path] = self.index_topic_id
-        url_map[self.index_topic_id] = home_path
-
-        return url_map, warnings
-
-    @cached_property
-    def notification_template(self):
-        notification_html = (
-            "<div class='{{ notification_class }}'>"
-            "<div class='p-notification__content'>"
-            "{{ contents | safe }}"
-            "</div></div>"
-        )
-
-        return Template(notification_html)
-
-    def _replace_notifications(self, soup):
-        """
-        Given some BeautifulSoup of a document,
-        replace blockquotes with the appropriate notification markup.
-
-        E.g. the following Markdown in a Discourse topic:
-
-            > ⓘ Content
-
-        Will generate the following markup, as per the CommonMark spec
-        (https://spec.commonmark.org/0.29/#block-quotes):
-
-            <blockquote><p>ⓘ Content</p></blockquote>
-
-        Becomes:
-
-            <div class="p-notification">
-                <div class="p-notification__content">
-                    <p class="u-no-padding--top u-no-margin--bottom">
-                        Content
-                    </p>
-                </div>
-            </div>
-        """
-
-        for note_string in soup.findAll(text=re.compile("ⓘ ")):
-            first_paragraph = note_string.parent
-            blockquote = first_paragraph.parent
-            last_paragraph = blockquote.findChildren(recursive=False)[-1]
-
-            if first_paragraph.name == "p" and blockquote.name == "blockquote":
-                # Remove extra padding/margin
-                first_paragraph.attrs["class"] = "u-no-padding--top"
-                if last_paragraph.name == "p":
-                    if "class" in last_paragraph.attrs:
-                        last_paragraph.attrs["class"] += " u-no-margin--bottom"
-                    else:
-                        last_paragraph.attrs["class"] = "u-no-margin--bottom"
-
-                # Remove control emoji
-                notification_html = blockquote.encode_contents().decode(
-                    "utf-8"
-                )
-                notification_html = re.sub(
-                    r"^\n?<p([^>]*)>ⓘ +", r"<p\1>", notification_html
-                )
-
-                notification = self.notification_template.render(
-                    notification_class="p-notification",
-                    contents=notification_html,
-                )
-                blockquote.replace_with(
-                    BeautifulSoup(notification, features="lxml")
-                )
-
-        for warning in soup.findAll("img", title=":warning:"):
-            first_paragraph = warning.parent
-            blockquote = first_paragraph.parent
-            last_paragraph = blockquote.findChildren(recursive=False)[-1]
-
-            if first_paragraph.name == "p" and blockquote.name == "blockquote":
-                warning.decompose()
-
-                # Remove extra padding/margin
-                first_paragraph.attrs["class"] = "u-no-padding--top"
-                if last_paragraph.name == "p":
-                    if "class" in last_paragraph.attrs:
-                        last_paragraph.attrs["class"] += " u-no-margin--bottom"
-                    else:
-                        last_paragraph.attrs["class"] = "u-no-margin--bottom"
-
-                # Strip leading space
-                first_item = last_paragraph.contents[0]
-                if isinstance(first_item, NavigableString):
-                    first_item.replace_with(first_item.lstrip(" "))
-
-                notification = self.notification_template.render(
-                    notification_class="p-notification--caution",
-                    contents=blockquote.encode_contents().decode("utf-8"),
-                )
-
-                blockquote.replace_with(
-                    BeautifulSoup(notification, features="lxml")
-                )
-
-        return soup
-
-    def _get_preamble(self, soup, break_on_title):
-        """
-        Given a BeautifulSoup HTML document, separate out the HTML
-        at the start, up to the heading defined in `break_on_title`,
-        and return it.
-        """
-
-        heading = soup.find(HEADER_REGEX, text=break_on_title)
-
-        if not heading:
-            return soup
-
-        return "".join(map(str, reversed(list(heading.previous_siblings))))
-
-    def parse_topic(self, topic, topic_soup=None):
-        """
-        Parse a topic object from the Discourse API
-        and return document data:
-        - title: The title
-        - body_html: The HTML content of the initial topic post
-                        (with some post-processing)
-        - updated: A human-readable date, relative to now
-                    (e.g. "3 days ago")
-        - forum_link: The link to the original forum post
-        """
-        updated_datetime = dateutil.parser.parse(
-            topic["post_stream"]["posts"][0]["updated_at"]
-        )
-
-        topic_path = f"/t/{topic['slug']}/{topic['id']}"
-
-        if topic_soup is None:
-            topic_soup = BeautifulSoup(
-                topic["post_stream"]["posts"][0]["cooked"], features="lxml"
-            )
-
-        soup = self._process_topic_soup(topic_soup)
-        self._replace_lightbox(soup)
-
-        return {
-            "title": topic["title"],
-            "body_soup": soup,
-            "body_html": str(soup),
-            "updated": humanize.naturaltime(
-                updated_datetime.replace(tzinfo=None)
-            ),
-            "topic_path": topic_path,
-        }
-
-    def _get_section(self, soup, title_text, content_tag=None):
-        """
-        Given some HTML soup and the text of a title within it,
-        get the content between that title and the next title
-        of the same level, and return it as another soup object.
-
-        E.g. if `soup` contains is:
-
-        <p>Pre</p>
-        <h2>My heading</h2>
-        <p>Content</p>
-        <h2>Next heading</h2>
-
-        and `title_text` is "My heading", then it will return:
-
-        <p>Content</p>
-        """
-        heading = soup.find(HEADER_REGEX, text=title_text)
-
-        if not heading:
-            return None
-
-        if content_tag:
-            return heading.find_next_sibling(content_tag)
-        html = str(soup)
-        section_html = html.split(str(heading))[1]
-
-        if f"<{heading.name}>" in html:
-            section_html = section_html.split(f"<{heading.name}>")[0]
-
-        return BeautifulSoup(section_html, features="lxml")
diff --git a/webapp/openapi_parser.py b/webapp/openapi_parser.py
index 51dc2025..96fa1b47 100644
--- a/webapp/openapi_parser.py
+++ b/webapp/openapi_parser.py
@@ -37,4 +37,5 @@ def read_yaml_from_url(url: str, session) -> str:
 
 
 def read_yaml_from_file(filename: str) -> str:
-    return open(filename).read()
+    with open(filename) as fh:
+        return fh.read()
diff --git a/webapp/views/__init__.py b/webapp/views/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/webapp/views/docs.py b/webapp/views/docs.py
new file mode 100644
index 00000000..69c5fbe0
--- /dev/null
+++ b/webapp/views/docs.py
@@ -0,0 +1,44 @@
+from os import getenv
+
+from canonicalwebteam.discourse import (
+    DiscourseAPI,
+    DocParser,
+    Docs,
+)
+from canonicalwebteam.search import build_search_view
+
+DISCOURSE_API_KEY = getenv("DISCOURSE_API_KEY")
+DISCOURSE_API_USERNAME = getenv("DISCOURSE_API_USERNAME")
+
+
+def init_docs(app, url_prefix: str, session) -> Docs:
+    """Initialise Docs attaching to Discourse."""
+    discourse_docs = Docs(
+        parser=DocParser(
+            api=DiscourseAPI(
+                base_url="https://discourse.maas.io/",
+                session=session,
+                api_key=DISCOURSE_API_KEY,
+                api_username=DISCOURSE_API_USERNAME,
+                get_topics_query_id=2,
+            ),
+            index_topic_id=6662,
+            url_prefix=url_prefix,
+            tutorials_index_topic_id=1289,
+            tutorials_url_prefix="/tutorials",
+        ),
+        document_template="docs/document.html",
+        url_prefix=url_prefix,
+    )
+    discourse_docs.init_app(app)
+
+    app.add_url_rule(
+        "/docs/search",
+        "docs-search",
+        build_search_view(
+            session=session,
+            site="maas.io/docs",
+            template_path="docs/search.html",
+        ),
+    )
+    return discourse_docs
diff --git a/webapp/views/tutorials.py b/webapp/views/tutorials.py
new file mode 100644
index 00000000..0b4ae303
--- /dev/null
+++ b/webapp/views/tutorials.py
@@ -0,0 +1,70 @@
+from os import getenv
+
+import flask
+
+from canonicalwebteam.discourse import DiscourseAPI, TutorialParser, Tutorials
+
+DISCOURSE_API_KEY = getenv("DISCOURSE_API_KEY")
+DISCOURSE_API_USERNAME = getenv("DISCOURSE_API_USERNAME")
+
+
+def init_tutorials(app, url_prefix, session):
+    tutorials_discourse = Tutorials(
+        parser=TutorialParser(
+            api=DiscourseAPI(
+                base_url="https://discourse.maas.io/",
+                session=session,
+                api_key=DISCOURSE_API_KEY,
+                api_username=DISCOURSE_API_USERNAME,
+                get_topics_query_id=2,
+            ),
+            index_topic_id=1289,
+            url_prefix=url_prefix,
+        ),
+        document_template="tutorials/tutorial.html",
+        url_prefix=url_prefix,
+        blueprint_name="tutorials",
+    )
+
+    @app.route(url_prefix)
+    def index():
+        tutorials_discourse.parser.ensure_parsed()
+        tutorials_discourse.parser.parse_topic(
+            tutorials_discourse.parser.index_topic
+        )
+        tutorials = tutorials_discourse.parser.tutorials
+        topic_list = []
+
+        for item in tutorials:
+            if item["categories"] not in topic_list:
+                topic_list.append(item["categories"])
+            item["categories"] = {
+                "slug": item["categories"],
+                "name": " ".join(
+                    [
+                        word.capitalize()
+                        for word in item["categories"].split("-")
+                    ]
+                ),
+            }
+
+        topic_list.sort()
+        topics = []
+
+        for topic in topic_list:
+            topics.append(
+                {
+                    "slug": topic,
+                    "name": " ".join(
+                        [word.capitalize() for word in topic.split("-")]
+                    ),
+                }
+            )
+
+        return flask.render_template(
+            "tutorials/index.html",
+            tutorials=tutorials,
+            topics=topics,
+        )
+
+    tutorials_discourse.init_app(app)