Unit test that checks that all plugin hooks have corresponding unit tests

[simonw]

Turns out some hooks are missing unit test coverage: #581 (comment)_

[simonw]

This seems to work:

diff --git a/tests/test_plugins.py b/tests/test_plugins.py
index 8b6a6b4..e9a40aa 100644
--- a/tests/test_plugins.py
+++ b/tests/test_plugins.py
@@ -7,7 +7,7 @@ from .fixtures import (
     TestClient as _TestClient,
 )  # noqa
 from datasette.app import Datasette
-from datasette.plugins import get_plugins, DEFAULT_PLUGINS
+from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm
 from datasette.utils import sqlite3
 import base64
 import json
@@ -20,6 +20,21 @@ import pytest
 import urllib
 
 
+def test_plugin_hooks_have_tests():
+    "Every plugin hook should be referenced in this test module"
+    hooks = [name for name in dir(pm.hook) if not name.startswith("_")]
+    tests_in_this_module = [t for t in globals().keys() if t.startswith('test_')]
+    untested = []
+    for hook in hooks:
+        ok = False
+        for test in tests_in_this_module:
+            if hook in test:
+                ok = True
+        if not ok:
+            untested.append(hook)
+    assert not untested, 'These plugin hooks are missing tests: {}'.format(untested)
+
+
 def test_plugins_dir_plugin_prepare_connection(app_client):
     response = app_client.get(
         "/fixtures.json?sql=select+convert_units(100%2C+'m'%2C+'ft')"

Based on how the documentation unit tests work.

Currently fails with:

AssertionError: These plugin hooks are missing tests:
['prepare_jinja2_environment', 'publish_subcommand', 'register_facet_classes', 'register_output_renderer']

[simonw]

I'll do the work for this in the pull request #772.

[simonw]

Actually I'll land this using @pytest.mark.xfail.

[simonw]

$ pytest -k test_plugin_hooks_have_tests -vv
====================================== test session starts ======================================
platform darwin -- Python 3.7.7, pytest-5.2.4, py-1.8.1, pluggy-0.13.1 -- /Users/simon/.local/share/virtualenvs/datasette-AWNrQs95/bin/python
cachedir: .pytest_cache
rootdir: /Users/simon/Dropbox/Development/datasette, inifile: pytest.ini
plugins: asyncio-0.10.0
collected 486 items / 475 deselected / 11 selected                                              

tests/test_plugins.py::test_plugin_hooks_have_tests[asgi_wrapper] XPASS                   [  9%]
tests/test_plugins.py::test_plugin_hooks_have_tests[extra_body_script] XPASS              [ 18%]
tests/test_plugins.py::test_plugin_hooks_have_tests[extra_css_urls] XPASS                 [ 27%]
tests/test_plugins.py::test_plugin_hooks_have_tests[extra_js_urls] XPASS                  [ 36%]
tests/test_plugins.py::test_plugin_hooks_have_tests[extra_template_vars] XPASS            [ 45%]
tests/test_plugins.py::test_plugin_hooks_have_tests[prepare_connection] XPASS             [ 54%]
tests/test_plugins.py::test_plugin_hooks_have_tests[prepare_jinja2_environment] XFAIL     [ 63%]
tests/test_plugins.py::test_plugin_hooks_have_tests[publish_subcommand] XFAIL             [ 72%]
tests/test_plugins.py::test_plugin_hooks_have_tests[register_facet_classes] XFAIL         [ 81%]
tests/test_plugins.py::test_plugin_hooks_have_tests[register_output_renderer] XFAIL       [ 90%]
tests/test_plugins.py::test_plugin_hooks_have_tests[render_cell] XPASS                    [100%]

========================= 475 deselected, 4 xfailed, 7 xpassed in 1.70s =========================

[simonw]

Closed in da87e96