diff --git a/datasette/filters.py b/datasette/filters.py index cbd9441555..5ea3488bd7 100644 --- a/datasette/filters.py +++ b/datasette/filters.py @@ -1,7 +1,172 @@ +from datasette import hookimpl +from datasette.views.base import DatasetteError +from datasette.utils.asgi import BadRequest import json import numbers +from .utils import detect_json1, escape_sqlite, path_with_removed_args -from .utils import detect_json1, escape_sqlite + +@hookimpl(specname="filters_from_request") +def where_filters(request, database, datasette): + # This one deals with ?_where= + async def inner(): + where_clauses = [] + extra_wheres_for_ui = [] + if "_where" in request.args: + if not await datasette.permission_allowed( + request.actor, + "execute-sql", + resource=database, + default=True, + ): + raise DatasetteError("_where= is not allowed", status=403) + else: + where_clauses.extend(request.args.getlist("_where")) + extra_wheres_for_ui = [ + { + "text": text, + "remove_url": path_with_removed_args(request, {"_where": text}), + } + for text in request.args.getlist("_where") + ] + + return FilterArguments( + where_clauses, + extra_context={ + "extra_wheres_for_ui": extra_wheres_for_ui, + }, + ) + + return inner + + +@hookimpl(specname="filters_from_request") +def search_filters(request, database, table, datasette): + # ?_search= and _search_colname= + async def inner(): + where_clauses = [] + params = {} + human_descriptions = [] + extra_context = {} + + # Figure out which fts_table to use + table_metadata = datasette.table_metadata(database, table) + db = datasette.get_database(database) + fts_table = request.args.get("_fts_table") + fts_table = fts_table or table_metadata.get("fts_table") + fts_table = fts_table or await db.fts_table(table) + fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) + search_args = { + key: request.args[key] + for key in request.args + if key.startswith("_search") and key != "_searchmode" + } + search = "" + search_mode_raw = table_metadata.get("searchmode") == "raw" + # Or set search mode from the querystring + qs_searchmode = request.args.get("_searchmode") + if qs_searchmode == "escaped": + search_mode_raw = False + if qs_searchmode == "raw": + search_mode_raw = True + + extra_context["supports_search"] = bool(fts_table) + + if fts_table and search_args: + if "_search" in search_args: + # Simple ?_search=xxx + search = search_args["_search"] + where_clauses.append( + "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format( + fts_table=escape_sqlite(fts_table), + fts_pk=escape_sqlite(fts_pk), + match_clause=":search" + if search_mode_raw + else "escape_fts(:search)", + ) + ) + human_descriptions.append(f'search matches "{search}"') + params["search"] = search + extra_context["search"] = search + else: + # More complex: search against specific columns + for i, (key, search_text) in enumerate(search_args.items()): + search_col = key.split("_search_", 1)[1] + if search_col not in await db.table_columns(fts_table): + raise BadRequest("Cannot search by that column") + + where_clauses.append( + "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( + fts_table=escape_sqlite(fts_table), + search_col=escape_sqlite(search_col), + match_clause=":search_{}".format(i) + if search_mode_raw + else "escape_fts(:search_{})".format(i), + ) + ) + human_descriptions.append( + f'search column "{search_col}" matches "{search_text}"' + ) + params[f"search_{i}"] = search_text + extra_context["search"] = search_text + + return FilterArguments(where_clauses, params, human_descriptions, extra_context) + + return inner + + +@hookimpl(specname="filters_from_request") +def through_filters(request, database, table, datasette): + # ?_search= and _search_colname= + async def inner(): + where_clauses = [] + params = {} + human_descriptions = [] + extra_context = {} + + # Support for ?_through={table, column, value} + if "_through" in request.args: + for through in request.args.getlist("_through"): + through_data = json.loads(through) + through_table = through_data["table"] + other_column = through_data["column"] + value = through_data["value"] + db = datasette.get_database(database) + outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) + try: + fk_to_us = [ + fk for fk in outgoing_foreign_keys if fk["other_table"] == table + ][0] + except IndexError: + raise DatasetteError( + "Invalid _through - could not find corresponding foreign key" + ) + param = f"p{len(params)}" + where_clauses.append( + "{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})".format( + through_table=escape_sqlite(through_table), + our_pk=escape_sqlite(fk_to_us["other_column"]), + our_column=escape_sqlite(fk_to_us["column"]), + other_column=escape_sqlite(other_column), + param=param, + ) + ) + params[param] = value + human_descriptions.append(f'{through_table}.{other_column} = "{value}"') + + return FilterArguments(where_clauses, params, human_descriptions, extra_context) + + return inner + + +class FilterArguments: + def __init__( + self, where_clauses, params=None, human_descriptions=None, extra_context=None + ): + self.where_clauses = where_clauses + self.params = params or {} + self.human_descriptions = human_descriptions or [] + self.extra_context = extra_context or {} class Filter: diff --git a/datasette/hookspecs.py b/datasette/hookspecs.py index 1d4e3b27a1..8f4fecab47 100644 --- a/datasette/hookspecs.py +++ b/datasette/hookspecs.py @@ -89,6 +89,17 @@ def actor_from_request(datasette, request): """Return an actor dictionary based on the incoming request""" +@hookspec +def filters_from_request(request, database, table, datasette): + """ + Return datasette.filters.FilterArguments( + where_clauses=[str, str, str], + params={}, + human_descriptions=[str, str, str], + extra_context={} + ) based on the request""" + + @hookspec def permission_allowed(datasette, actor, action, resource): """Check if actor is allowed to perform this action - return True, False or None""" diff --git a/datasette/plugins.py b/datasette/plugins.py index 50791988e0..76b46a4737 100644 --- a/datasette/plugins.py +++ b/datasette/plugins.py @@ -8,6 +8,7 @@ "datasette.publish.heroku", "datasette.publish.cloudrun", "datasette.facets", + "datasette.filters", "datasette.sql_functions", "datasette.actor_auth_cookie", "datasette.default_permissions", diff --git a/datasette/views/table.py b/datasette/views/table.py index da2639669b..cfd31bd382 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -442,117 +442,27 @@ async def data( filters = Filters(sorted(other_args), units, ureg) where_clauses, params = filters.build_where_clauses(table) - extra_wheres_for_ui = [] - # Add _where= from querystring - if "_where" in request.args: - if not await self.ds.permission_allowed( - request.actor, - "execute-sql", - resource=database, - default=True, - ): - raise DatasetteError("_where= is not allowed", status=403) - else: - where_clauses.extend(request.args.getlist("_where")) - extra_wheres_for_ui = [ - { - "text": text, - "remove_url": path_with_removed_args(request, {"_where": text}), - } - for text in request.args.getlist("_where") - ] - - # Support for ?_through={table, column, value} + # Execute filters_from_request plugin hooks + extra_context_from_filters = {} extra_human_descriptions = [] - if "_through" in request.args: - for through in request.args.getlist("_through"): - through_data = json.loads(through) - through_table = through_data["table"] - other_column = through_data["column"] - value = through_data["value"] - outgoing_foreign_keys = await db.foreign_keys_for_table(through_table) - try: - fk_to_us = [ - fk for fk in outgoing_foreign_keys if fk["other_table"] == table - ][0] - except IndexError: - raise DatasetteError( - "Invalid _through - could not find corresponding foreign key" - ) - param = f"p{len(params)}" - where_clauses.append( - "{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})".format( - through_table=escape_sqlite(through_table), - our_pk=escape_sqlite(fk_to_us["other_column"]), - our_column=escape_sqlite(fk_to_us["column"]), - other_column=escape_sqlite(other_column), - param=param, - ) - ) - params[param] = value - extra_human_descriptions.append( - f'{through_table}.{other_column} = "{value}"' - ) - - # _search= support: - fts_table = special_args.get("_fts_table") - fts_table = fts_table or table_metadata.get("fts_table") - fts_table = fts_table or await db.fts_table(table) - fts_pk = special_args.get("_fts_pk", table_metadata.get("fts_pk", "rowid")) - search_args = dict( - pair - for pair in special_args.items() - if pair[0].startswith("_search") and pair[0] != "_searchmode" - ) - search = "" - search_mode_raw = table_metadata.get("searchmode") == "raw" - # Or set it from the querystring - qs_searchmode = special_args.get("_searchmode") - if qs_searchmode == "escaped": - search_mode_raw = False - if qs_searchmode == "raw": - search_mode_raw = True - if fts_table and search_args: - if "_search" in search_args: - # Simple ?_search=xxx - search = search_args["_search"] - where_clauses.append( - "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format( - fts_table=escape_sqlite(fts_table), - fts_pk=escape_sqlite(fts_pk), - match_clause=":search" - if search_mode_raw - else "escape_fts(:search)", - ) - ) - extra_human_descriptions.append(f'search matches "{search}"') - params["search"] = search - else: - # More complex: search against specific columns - for i, (key, search_text) in enumerate(search_args.items()): - search_col = key.split("_search_", 1)[1] - if search_col not in await db.table_columns(fts_table): - raise BadRequest("Cannot search by that column") - - where_clauses.append( - "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format( - fts_table=escape_sqlite(fts_table), - search_col=escape_sqlite(search_col), - match_clause=":search_{}".format(i) - if search_mode_raw - else "escape_fts(:search_{})".format(i), - ) - ) - extra_human_descriptions.append( - f'search column "{search_col}" matches "{search_text}"' - ) - params[f"search_{i}"] = search_text + for hook in pm.hook.filters_from_request( + request=request, + table=table, + database=database, + datasette=self.ds, + ): + filter_arguments = await await_me_maybe(hook) + if filter_arguments: + where_clauses.extend(filter_arguments.where_clauses) + params.update(filter_arguments.params) + extra_human_descriptions.extend(filter_arguments.human_descriptions) + extra_context_from_filters.update(filter_arguments.extra_context) + + # Deal with custom sort orders sortable_columns = await self.sortable_columns_for_table( database, table, use_rowid ) - - # Allow for custom sort order sort = special_args.get("_sort") sort_desc = special_args.get("_sort_desc") @@ -942,10 +852,8 @@ async def table_actions(): for table_column in table_columns if table_column not in columns ] - return { + d = { "table_actions": table_actions, - "supports_search": bool(fts_table), - "search": search or "", "use_rowid": use_rowid, "filters": filters, "display_columns": display_columns, @@ -957,7 +865,6 @@ async def table_actions(): key=lambda f: (len(f["results"]), f["name"]), reverse=True, ), - "extra_wheres_for_ui": extra_wheres_for_ui, "form_hidden_args": form_hidden_args, "is_sortable": any(c["sortable"] for c in display_columns), "fix_path": self.ds.urls.path, @@ -977,6 +884,8 @@ async def table_actions(): "view_definition": await db.get_view_definition(table), "table_definition": await db.get_table_definition(table), } + d.update(extra_context_from_filters) + return d return ( { diff --git a/docs/plugin_hooks.rst b/docs/plugin_hooks.rst index 4a7c36c303..d76f70e50a 100644 --- a/docs/plugin_hooks.rst +++ b/docs/plugin_hooks.rst @@ -923,6 +923,59 @@ Instead of returning a dictionary, this function can return an awaitable functio Example: `datasette-auth-tokens `_ +.. _plugin_hook_filters_from_request: + +filters_from_request(request, database, table, datasette) +--------------------------------------------------------- + +``request`` - object + The current HTTP :ref:`internals_request`. + +``database`` - string + The name of the database. + +``table`` - string + The name of the table. + +``datasette`` - :ref:`internals_datasette` + You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``, or to execute SQL queries. + +This hook runs on the :ref:`table ` page, and can influence the ``where`` clause of the SQL query used to populate that page, based on query string arguments on the incoming request. + +The hook should return an instance of ``datasette.filters.FilterArguments`` which has one required and three optional arguments: + +.. code-block:: python + + return FilterArguments( + where_clauses=["id > :max_id"], + params={"max_id": 5}, + human_descriptions=["max_id is greater than 5"], + extra_context={} + ) + +The arguments to the ``FilterArguments`` class constructor are as follows: + +``where_clauses`` - list of strings, required + A list of SQL fragments that will be inserted into the SQL query, joined by the ``and`` operator. These can include ``:named`` parameters which will be populated using data in ``params``. +``params`` - dictionary, optional + Additional keyword arguments to be used when the query is executed. These should match any ``:arguments`` in the where clauses. +``human_descriptions`` - list of strings, optional + These strings will be included in the human-readable description at the top of the page and the page ````. +``extra_context`` - dictionary, optional + Additional context variables that should be made available to the ``table.html`` template when it is rendered. + +This example plugin causes 0 results to be returned if ``?_nothing=1`` is added to the URL: + +.. code-block:: python + + from datasette import hookimpl + from datasette.filters import FilterArguments + + @hookimpl + def filters_from_request(self, request): + if request.args.get("_nothing"): + return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"]) + .. _plugin_hook_permission_allowed: permission_allowed(datasette, actor, action, resource) diff --git a/tests/test_filters.py b/tests/test_filters.py index d05ae80fae..2ff57489b3 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -1,4 +1,6 @@ -from datasette.filters import Filters +from datasette.filters import Filters, through_filters, where_filters, search_filters +from datasette.utils.asgi import Request +from .fixtures import app_client import pytest @@ -74,3 +76,86 @@ def test_build_where(args, expected_where, expected_params): sql_bits, actual_params = f.build_where_clauses("table") assert expected_where == sql_bits assert {f"p{i}": param for i, param in enumerate(expected_params)} == actual_params + + +@pytest.mark.asyncio +async def test_through_filters_from_request(app_client): + request = Request.fake( + '/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' + ) + filter_args = await ( + through_filters( + request=request, + datasette=app_client.ds, + table="roadside_attractions", + database="fixtures", + ) + )() + assert filter_args.where_clauses == [ + "pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)" + ] + assert filter_args.params == {"p0": "1"} + assert filter_args.human_descriptions == [ + 'roadside_attraction_characteristics.characteristic_id = "1"' + ] + assert filter_args.extra_context == {} + + +@pytest.mark.asyncio +async def test_through_filters_from_request(app_client): + request = Request.fake( + '/?_through={"table":"roadside_attraction_characteristics","column":"characteristic_id","value":"1"}' + ) + filter_args = await ( + through_filters( + request=request, + datasette=app_client.ds, + table="roadside_attractions", + database="fixtures", + ) + )() + assert filter_args.where_clauses == [ + "pk in (select attraction_id from roadside_attraction_characteristics where characteristic_id = :p0)" + ] + assert filter_args.params == {"p0": "1"} + assert filter_args.human_descriptions == [ + 'roadside_attraction_characteristics.characteristic_id = "1"' + ] + assert filter_args.extra_context == {} + + +@pytest.mark.asyncio +async def test_where_filters_from_request(app_client): + request = Request.fake("/?_where=pk+>+3") + filter_args = await ( + where_filters( + request=request, + datasette=app_client.ds, + database="fixtures", + ) + )() + assert filter_args.where_clauses == ["pk > 3"] + assert filter_args.params == {} + assert filter_args.human_descriptions == [] + assert filter_args.extra_context == { + "extra_wheres_for_ui": [{"text": "pk > 3", "remove_url": "/"}] + } + + +@pytest.mark.asyncio +async def test_search_filters_from_request(app_client): + request = Request.fake("/?_search=bobcat") + filter_args = await ( + search_filters( + request=request, + datasette=app_client.ds, + database="fixtures", + table="searchable", + ) + )() + assert filter_args.where_clauses == [ + "rowid in (select rowid from searchable_fts where searchable_fts match escape_fts(:search))" + ] + assert filter_args.params == {"search": "bobcat"} + assert filter_args.human_descriptions == ['search matches "bobcat"'] + assert filter_args.extra_context == {"supports_search": True, "search": "bobcat"} diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 1da284536c..656f39e47a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -9,6 +9,7 @@ from click.testing import CliRunner from datasette.app import Datasette from datasette import cli, hookimpl +from datasette.filters import FilterArguments from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm from datasette.utils.sqlite import sqlite3 from datasette.utils import CustomRow @@ -977,3 +978,20 @@ def unverify(): } pm.unregister(name="verify") importlib.reload(cli) + + +def test_hook_filters_from_request(app_client): + class ReturnNothingPlugin: + __name__ = "ReturnNothingPlugin" + + @hookimpl + def filters_from_request(self, request): + if request.args.get("_nothing"): + return FilterArguments(["1 = 0"], human_descriptions=["NOTHING"]) + + pm.register(ReturnNothingPlugin(), name="ReturnNothingPlugin") + response = app_client.get("/fixtures/facetable?_nothing=1") + assert "0 rows\n where NOTHING" in response.text + json_response = app_client.get("/fixtures/facetable.json?_nothing=1") + assert json_response.json["rows"] == [] + pm.unregister(name="ReturnNothingPlugin")