From 3aef8ac91015d04536d96b8013e6907a1a2d3e1c Mon Sep 17 00:00:00 2001 From: Johan Bisse Mattsson Date: Wed, 18 Dec 2024 10:17:27 +0100 Subject: [PATCH] feat(lxl-web, supersearch): Narrow search query when editing parts (LWS-273) (#1190) * Get editedRanges by parsing the syntax tree server-side * Add tests * Only do search request if trimmed value isn't empty * Capitalize BooleanOperator rule in grammar (needed to be able to identify the operators) * Narrow down search query when editing qualifier parts * Keep _q as is for now (but add _qualifier param) --- lxl-web/src/lib/components/Search.svelte | 10 +- .../api/[[lang=lang]]/supersearch/+server.ts | 25 +++- .../supersearch/getEditedPartEntries.test.ts | 20 +++ .../supersearch/getEditedPartEntries.ts | 65 ++++++++++ .../supersearch/getEditedRanges.test.ts | 119 ++++++++++++++++++ .../supersearch/getEditedRanges.ts | 84 +++++++++++++ .../src/syntax.grammar | 8 +- .../codemirror-lang-lxlquery/test/cases.txt | 9 +- .../src/lib/components/SuperSearch.svelte | 6 +- .../supersearch/src/lib/types/superSearch.ts | 2 +- .../src/lib/utils/useSearchRequest.svelte.ts | 9 +- 11 files changed, 338 insertions(+), 19 deletions(-) create mode 100644 lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.test.ts create mode 100644 lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.ts create mode 100644 lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.test.ts create mode 100644 lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.ts diff --git a/lxl-web/src/lib/components/Search.svelte b/lxl-web/src/lib/components/Search.svelte index 7f34d9f5a..5dc29463e 100644 --- a/lxl-web/src/lib/components/Search.svelte +++ b/lxl-web/src/lib/components/Search.svelte @@ -64,11 +64,13 @@ language={lxlQuery} placeholder={$page.data.t('search.search')} endpoint={'/api/supersearch'} - queryFn={(query) => - new URLSearchParams({ + queryFn={(query, cursor) => { + return new URLSearchParams({ _q: query, - _limit: '10' - })} + _limit: '10', + cursor: cursor.toString() + }); + }} paginationQueryFn={handlePaginationQuery} extensions={[lxlQualifierPlugin]} > diff --git a/lxl-web/src/routes/api/[[lang=lang]]/supersearch/+server.ts b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/+server.ts index f49ba421f..2a156b87f 100644 --- a/lxl-web/src/routes/api/[[lang=lang]]/supersearch/+server.ts +++ b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/+server.ts @@ -4,12 +4,35 @@ import type { RequestHandler } from './$types.ts'; import { LxlLens } from '$lib/types/display'; import { getSupportedLocale } from '$lib/i18n/locales.js'; import { toString } from '$lib/utils/xl.js'; +import getEditedPartEntries from './getEditedPartEntries.js'; + +/** + * TODO: + * - Investigate how we should also send the full query if we wish to boost faceted results. + * - Investigate if we also should do a separate query for the last edited word – and not only the whole phrase (e.g. should we show a result for the subject `winter` when entering `astrid lindgren winter`?) + */ export const GET: RequestHandler = async ({ url, params, locals }) => { const displayUtil = locals.display; const locale = getSupportedLocale(params?.lang); - const findResponse = await fetch(`${env.API_URL}/find?${url.searchParams.toString()}`); + const _q = url.searchParams.get('_q'); + const cursor = parseInt(url.searchParams.get('cursor') || '0', 10); + + const newSearchParams = new URLSearchParams([...Array.from(url.searchParams.entries())]); + + if (_q && Number.isInteger(cursor)) { + const editedPartEntries = getEditedPartEntries(_q, cursor); + + editedPartEntries.forEach(([key, value]) => { + newSearchParams.set(key, value); + }); + newSearchParams.delete('cursor'); + console.log('Initial search params:', decodeURIComponent(url.searchParams.toString())); + console.log('Search params sent to /find:', decodeURIComponent(newSearchParams.toString())); + } + + const findResponse = await fetch(`${env.API_URL}/find?${newSearchParams.toString()}`); const data = await findResponse.json(); return json({ diff --git a/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.test.ts b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.test.ts new file mode 100644 index 000000000..6ad409520 --- /dev/null +++ b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.test.ts @@ -0,0 +1,20 @@ +import { describe, it, expect } from 'vitest'; +import getEditedPartEntries from './getEditedPartEntries'; + +describe('getEditedPartEntries', () => { + it('narrows down search query when editing qualifier parts', () => { + expect(getEditedPartEntries('hello title:"hej"', 16)).toEqual([['_qualifier', 'title:"hej"']]); + }); + it('keeps query as is when editing year qualifiers', () => { + expect(getEditedPartEntries('hello ÅR:2024', 13)).toEqual([]); + }); + it('narrows down search query by base class for query codes', () => { + expect(getEditedPartEntries('astrid lindgren subject:"winter"', 27)).toEqual([ + ['_qualifier', `"rdf:type":Topic "winter"`], + ['min-reverseLinks.totalItems', '1'] + ]); + }); + it('otherwise keeps query as is', () => { + expect(getEditedPartEntries('hello', 5)).toEqual([]); + }); +}); diff --git a/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.ts b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.ts new file mode 100644 index 000000000..ca5f03570 --- /dev/null +++ b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.ts @@ -0,0 +1,65 @@ +import getEditedRanges from './getEditedRanges.js'; + +/** + * TODO: How should we handle translated query codes and qualifier keys? + */ + +const QUALIFIER_KEY_BY_BASE_CLASS = { + Library: 'itemHeldBy', + Agent: 'contributor', + Topic: 'subject', + Subject: 'subject', + Language: 'SPRÅK', + GenreForm: 'genreForm', + Person: 'person', + Work: 'titel' +}; + +const SKIP_QUALIFIERS = ['år']; + +/** + * Gets the URLSearchParams entries which should be appended/replaced with new values when editing a part of a query. + */ + +function getEditedPartEntries(query: string, cursor: number): [string, string][] { + const editedRanges = getEditedRanges(query, cursor); + + /** + * Narrow down search query when editing qualifier parts + */ + if (editedRanges.qualifierKey && editedRanges.qualifierOperator && editedRanges.qualifierValue) { + const qualifierKey = query.slice(editedRanges.qualifierKey.from, editedRanges.qualifierKey.to); + const qualifierOperator = query.slice( + editedRanges.qualifierOperator.from, + editedRanges.qualifierOperator.to + ); + const qualifierValue = query.slice( + editedRanges.qualifierValue.from, + editedRanges.qualifierValue.to + ); + + if (SKIP_QUALIFIERS.includes(qualifierKey.toLowerCase())) { + return []; // Keep query as is when editing year qualifiers + } + + const baseClass = Object.entries(QUALIFIER_KEY_BY_BASE_CLASS).find( + ([, key]) => key === qualifierKey + )?.[0]; + + if (baseClass) { + return [ + ['_qualifier', `"rdf:type":${baseClass} ${qualifierValue}`], + ['min-reverseLinks.totalItems', '1'] // ensure results are linked/used atleast once + ]; + } + + return [['_qualifier', qualifierKey + qualifierOperator + qualifierValue]]; + } + + /** + * Otherwise keep query entries as is + */ + return []; +} + +export default getEditedPartEntries; diff --git a/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.test.ts b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.test.ts new file mode 100644 index 000000000..57e833d65 --- /dev/null +++ b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.test.ts @@ -0,0 +1,119 @@ +import { describe, it, expect } from 'vitest'; +import getEditedRanges from './getEditedRanges'; + +describe('getEditedRanges', () => { + it('calculates the edited range for a simple free-text query', () => { + const query = 'hello'; + const editedRanges = getEditedRanges(query, 5); + expect(editedRanges).toEqual({ from: 0, to: 5 }); + expect(query.slice(editedRanges.from, editedRanges.to)).toBe('hello'); + }); + + it('calculates the edited range (with included ranges of qualifier parts) when editing a qualifier', () => { + const query = 'hasTitle:"a"'; + const editedRanges = getEditedRanges(query, 11); + expect(editedRanges).toEqual({ + from: 0, + to: 12, + qualifierKey: { + from: 0, + to: 8 + }, + qualifierOperator: { + from: 8, + to: 9 + }, + qualifierValue: { + from: 9, + to: 12 + } + }); + expect(query.slice(editedRanges.from, editedRanges.to)).toBe('hasTitle:"a"'); + expect(query.slice(editedRanges.qualifierKey?.from, editedRanges.qualifierKey?.to)).toBe( + 'hasTitle' + ); + expect( + query.slice(editedRanges.qualifierOperator?.from, editedRanges.qualifierOperator?.to) + ).toBe(':'); + expect(query.slice(editedRanges.qualifierValue?.from, editedRanges.qualifierValue?.to)).toBe( + '"a"' + ); + }); + + it('calculates the edited range when editing a string', () => { + expect(getEditedRanges('"hello"', 6)).toEqual({ + from: 0, + to: 7 + }); + }); + + it('calculates the edited range when editing a group', () => { + expect(getEditedRanges('(hello world)', 9)).toEqual({ + from: 0, + to: 13 + }); + }); + + it('calculates the edited range when editing a part after a qualifier', () => { + expect(getEditedRanges('hasTitle:"a" hello', 18)).toEqual({ + from: 12, + to: 18 + }); + }); + + it('calculates the edited range when editing a part after a qualifier', () => { + expect(getEditedRanges('hasTitle:"a" hello', 18)).toEqual({ + from: 12, + to: 18 + }); + }); + + it('calculates the edited range when editing a part before a qualifier', () => { + expect(getEditedRanges('hello hasTitle:"a"', 5)).toEqual({ + from: 0, + to: 6 + }); + }); + + it('calculates the edited range when editing a part surrounded by qualifiers', () => { + expect(getEditedRanges('hasTitle:"a" hello hasTitle:"b', 18)).toEqual({ + from: 12, + to: 19 + }); + }); + + it('calculates the edited range when editing a part after a group', () => { + expect(getEditedRanges('(hi) hello', 7)).toEqual({ + from: 4, + to: 10 + }); + }); + + it('calculates the edited range when editing a part before a group', () => { + expect(getEditedRanges('hello (hi)', 3)).toEqual({ + from: 0, + to: 6 + }); + }); + + it('calculates the edited range when editing a part surrounded by groups', () => { + expect(getEditedRanges('(hi) hello (hej)', 7)).toEqual({ + from: 4, + to: 11 + }); + }); + + it('calculates the edited range when editing a part before a boolean operator', () => { + expect(getEditedRanges('hello AND world', 5)).toEqual({ + from: 0, + to: 6 + }); + }); + + it('calculates the edited range when editing a part after a boolean operator', () => { + expect(getEditedRanges('hello AND world', 12)).toEqual({ + from: 9, + to: 15 + }); + }); +}); diff --git a/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.ts b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.ts new file mode 100644 index 000000000..4a09510fb --- /dev/null +++ b/lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.ts @@ -0,0 +1,84 @@ +import { lxlQuery } from 'codemirror-lang-lxlquery'; + +type Range = { + from: number; + to: number; +}; + +export type EditedRanges = Range & { + qualifierKey?: Range; + qualifierOperator?: Range; + qualifierValue?: Range; +}; + +function getEditedRanges(query: string, cursor: number): EditedRanges { + const tree = lxlQuery.language.parser.parse(query); + const innerNode = tree.resolveInner(cursor); + + /** + * Return `from` and `to` from qualifier parts if editing qualifier value + */ + if (innerNode.parent?.type.is('QualifierValue')) { + const qualifierNode = innerNode.parent.parent; + const qualifierKeyNode = qualifierNode?.getChild('QualifierKey'); + const qualifierOperatorNode = qualifierNode?.getChild('QualifierOperator'); + const qualiferValueNode = qualifierNode?.getChild('QualifierValue'); + if (qualifierNode) { + return { + from: qualifierNode.from, + to: qualifierNode.to, + ...(qualifierKeyNode && { + qualifierKey: { from: qualifierKeyNode.from, to: qualifierKeyNode.to } + }), + ...(qualifierOperatorNode && { + qualifierOperator: { from: qualifierOperatorNode.from, to: qualifierOperatorNode.to } + }), + ...(qualiferValueNode && { + qualifierValue: { from: qualiferValueNode.from, to: qualiferValueNode.to } + }) + }; + } + } + + let from = 0; + let to = query.length; + + /** + * Adjust `from` and `to` if enclosed qualifiers or groups are found BEFORE the edited part + * */ + tree.iterate({ + from: 0, + to: cursor, + enter(node) { + if (node.type.is('Qualifier') || node.type.is('Group') || node.type.is('BooleanOperator')) { + if (node.to > cursor) { + from = node.from; + to = node.to; + } else { + from = node.to; + } + } + } + }); + + /** + * Adjust `from` and `to` if enclosed qualifiers or groups are found AFTER the edited part + * */ + tree.iterate({ + from, + to, + enter(node) { + if ( + (node.type.is('Qualifier') || node.type.is('Group') || node.type.is('BooleanOperator')) && + node.from > cursor && + node.from < to + ) { + to = node.from; + } + } + }); + + return { from, to }; +} + +export default getEditedRanges; diff --git a/packages/codemirror-lang-lxlquery/src/syntax.grammar b/packages/codemirror-lang-lxlquery/src/syntax.grammar index b87fefa16..8becf0758 100644 --- a/packages/codemirror-lang-lxlquery/src/syntax.grammar +++ b/packages/codemirror-lang-lxlquery/src/syntax.grammar @@ -12,8 +12,8 @@ term { Group { "(" term* ")" } BooleanQuery { - (freetext | Qualifier | Group ) booleanOperator (freetext | Qualifier | Group ) - (booleanOperator (freetext | Qualifier | Group ))+? + (freetext | Qualifier | Group ) BooleanOperator (freetext | Qualifier | Group ) + (BooleanOperator (freetext | Qualifier | Group ))+? } Qualifier { @@ -47,7 +47,7 @@ freetext { CompareOperator { ">" | "<" | ">=" | "<=" } - booleanOperator { "AND" | "OR" | "NOT" } + BooleanOperator { "AND" | "OR" | "NOT" } Wildcard { "*"+ } @@ -55,7 +55,7 @@ freetext { space { @whitespace+ } - @precedence { booleanOperator, reserved, Identifier } + @precedence { BooleanOperator, reserved, Identifier } } @detectDelim \ No newline at end of file diff --git a/packages/codemirror-lang-lxlquery/test/cases.txt b/packages/codemirror-lang-lxlquery/test/cases.txt index 1948bc332..21eff5635 100644 --- a/packages/codemirror-lang-lxlquery/test/cases.txt +++ b/packages/codemirror-lang-lxlquery/test/cases.txt @@ -89,7 +89,7 @@ sommar OR vinter NOT vår ==> Query( - BooleanQuery(Identifier, Identifier, Identifier) + BooleanQuery(Identifier, BooleanOperator, Identifier, BooleanOperator, Identifier) ) @@ -100,7 +100,7 @@ Query( ==> Query( - BooleanQuery(Group(...),Group(...)) + BooleanQuery(Group(...),BooleanOperator,Group(...)) ) @@ -110,7 +110,7 @@ OR AND sommar ==> -Query(⚠, ⚠, Identifier) +Query(⚠(BooleanOperator), ⚠(BooleanOperator), Identifier) # Combined: Qualifier and Freetext @@ -131,7 +131,7 @@ träd* bibliografi:"sigel:DST" NOT typ:Text ==> Query( - Identifier, Wildcard, BooleanQuery(Qualifier(QualifierKey(Identifier), QualifierOperator(EqualOperator), QualifierValue(String)), Qualifier(QualifierKey(Identifier), QualifierOperator(EqualOperator), QualifierValue(Identifier))) + Identifier, Wildcard, BooleanQuery(Qualifier(QualifierKey(Identifier), QualifierOperator(EqualOperator), QualifierValue(String)), BooleanOperator, Qualifier(QualifierKey(Identifier), QualifierOperator(EqualOperator), QualifierValue(Identifier))) ) @@ -149,6 +149,7 @@ Query( Group( BooleanQuery( Identifier, + BooleanOperator, Identifier ) ) diff --git a/packages/supersearch/src/lib/components/SuperSearch.svelte b/packages/supersearch/src/lib/components/SuperSearch.svelte index 2d47f42d5..9a57726e9 100644 --- a/packages/supersearch/src/lib/components/SuperSearch.svelte +++ b/packages/supersearch/src/lib/components/SuperSearch.svelte @@ -46,6 +46,7 @@ let collapsedEditorView: EditorView | undefined = $state(); let expandedEditorView: EditorView | undefined = $state(); let dialog: HTMLDialogElement | undefined = $state(); + let cursor: number = $state(0); let placeholderCompartment = new Compartment(); let prevPlaceholder = placeholder; @@ -58,8 +59,8 @@ }); $effect(() => { - if (value) { - search.debouncedFetchData(value); + if (value && value.trim()) { + search.debouncedFetchData(value, cursor); } }); @@ -81,6 +82,7 @@ showExpandedSearch(); } value = event.value; + cursor = event.cursor; } function showExpandedSearch() { diff --git a/packages/supersearch/src/lib/types/superSearch.ts b/packages/supersearch/src/lib/types/superSearch.ts index 06cc88bfe..1fb1e0aa7 100644 --- a/packages/supersearch/src/lib/types/superSearch.ts +++ b/packages/supersearch/src/lib/types/superSearch.ts @@ -1,6 +1,6 @@ import type { JSONValue } from './json.js'; -export type QueryFunction = (value: string) => URLSearchParams; +export type QueryFunction = (value: string, cursor: number) => URLSearchParams; export type PaginationQueryFunction = ( searchParams: URLSearchParams, data: JSONValue diff --git a/packages/supersearch/src/lib/utils/useSearchRequest.svelte.ts b/packages/supersearch/src/lib/utils/useSearchRequest.svelte.ts index ec88ab085..5c0722e26 100644 --- a/packages/supersearch/src/lib/utils/useSearchRequest.svelte.ts +++ b/packages/supersearch/src/lib/utils/useSearchRequest.svelte.ts @@ -56,14 +56,17 @@ export function useSearchRequest({ } } - async function fetchData(query: string) { - data = await _fetchData(queryFn(query)); + async function fetchData(query: string, cursor: number) { + data = await _fetchData(queryFn(query, cursor)); if (paginationQueryFn) { paginatedData = [data]; } } - const debouncedFetchData = debounce((query: string) => fetchData(query), debouncedWait); + const debouncedFetchData = debounce( + (query: string, cursor: number) => fetchData(query, cursor), + debouncedWait + ); async function fetchMoreData() { if (moreSearchParams) {