feat(lxl-web, supersearch): Narrow search query when editing parts (L…

…WS-273) (#1190)

* Get editedRanges by parsing the syntax tree server-side

* Add tests

* Only do search request if trimmed value isn't empty

* Capitalize BooleanOperator rule in grammar (needed to be able to identify the operators)

* Narrow down search query when editing qualifier parts

* Keep _q as is for now (but add _qualifier param)

lxl-web/src/lib/components/Search.svelte

@@ -64,11 +64,13 @@
 			language={lxlQuery}
 			placeholder={$page.data.t('search.search')}
 			endpoint={'/api/supersearch'}
-			queryFn={(query) =>
-				new URLSearchParams({
+			queryFn={(query, cursor) => {
+				return new URLSearchParams({
 					_q: query,
-					_limit: '10'
-				})}
+					_limit: '10',
+					cursor: cursor.toString()
+				});
+			}}
 			paginationQueryFn={handlePaginationQuery}
 			extensions={[lxlQualifierPlugin]}
 		>

lxl-web/src/routes/api/[[lang=lang]]/supersearch/+server.ts

@@ -4,12 +4,35 @@ import type { RequestHandler } from './$types.ts';
 import { LxlLens } from '$lib/types/display';
 import { getSupportedLocale } from '$lib/i18n/locales.js';
 import { toString } from '$lib/utils/xl.js';
+import getEditedPartEntries from './getEditedPartEntries.js';
+
+/**
+ * TODO:
+ * - Investigate how we should also send the full query if we wish to boost faceted results.
+ * - Investigate if we also should do a separate query for the last edited word – and not only the whole phrase (e.g. should we show a result for the subject `winter` when entering `astrid lindgren winter`?)
+ */
 
 export const GET: RequestHandler = async ({ url, params, locals }) => {
 	const displayUtil = locals.display;
 	const locale = getSupportedLocale(params?.lang);
 
-	const findResponse = await fetch(`${env.API_URL}/find?${url.searchParams.toString()}`);
+	const _q = url.searchParams.get('_q');
+	const cursor = parseInt(url.searchParams.get('cursor') || '0', 10);
+
+	const newSearchParams = new URLSearchParams([...Array.from(url.searchParams.entries())]);
+
+	if (_q && Number.isInteger(cursor)) {
+		const editedPartEntries = getEditedPartEntries(_q, cursor);
+
+		editedPartEntries.forEach(([key, value]) => {
+			newSearchParams.set(key, value);
+		});
+		newSearchParams.delete('cursor');
+		console.log('Initial search params:', decodeURIComponent(url.searchParams.toString()));
+		console.log('Search params sent to /find:', decodeURIComponent(newSearchParams.toString()));
+	}
+
+	const findResponse = await fetch(`${env.API_URL}/find?${newSearchParams.toString()}`);
 	const data = await findResponse.json();
 
 	return json({

lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.test.ts

@@ -0,0 +1,20 @@
+import { describe, it, expect } from 'vitest';
+import getEditedPartEntries from './getEditedPartEntries';
+
+describe('getEditedPartEntries', () => {
+	it('narrows down search query when editing qualifier parts', () => {
+		expect(getEditedPartEntries('hello title:"hej"', 16)).toEqual([['_qualifier', 'title:"hej"']]);
+	});
+	it('keeps query as is when editing year qualifiers', () => {
+		expect(getEditedPartEntries('hello ÅR:2024', 13)).toEqual([]);
+	});
+	it('narrows down search query by base class for query codes', () => {
+		expect(getEditedPartEntries('astrid lindgren subject:"winter"', 27)).toEqual([
+			['_qualifier', `"rdf:type":Topic "winter"`],
+			['min-reverseLinks.totalItems', '1']
+		]);
+	});
+	it('otherwise keeps query as is', () => {
+		expect(getEditedPartEntries('hello', 5)).toEqual([]);
+	});
+});

lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedPartEntries.ts

@@ -0,0 +1,65 @@
+import getEditedRanges from './getEditedRanges.js';
+
+/**
+ * TODO: How should we handle translated query codes and qualifier keys?
+ */
+
+const QUALIFIER_KEY_BY_BASE_CLASS = {
+	Library: 'itemHeldBy',
+	Agent: 'contributor',
+	Topic: 'subject',
+	Subject: 'subject',
+	Language: 'SPRÅK',
+	GenreForm: 'genreForm',
+	Person: 'person',
+	Work: 'titel'
+};
+
+const SKIP_QUALIFIERS = ['år'];
+
+/**
+ * Gets the URLSearchParams entries which should be appended/replaced with new values when editing a part of a query.
+ */
+
+function getEditedPartEntries(query: string, cursor: number): [string, string][] {
+	const editedRanges = getEditedRanges(query, cursor);
+
+	/**
+	 * Narrow down search query when editing qualifier parts
+	 */
+	if (editedRanges.qualifierKey && editedRanges.qualifierOperator && editedRanges.qualifierValue) {
+		const qualifierKey = query.slice(editedRanges.qualifierKey.from, editedRanges.qualifierKey.to);
+		const qualifierOperator = query.slice(
+			editedRanges.qualifierOperator.from,
+			editedRanges.qualifierOperator.to
+		);
+		const qualifierValue = query.slice(
+			editedRanges.qualifierValue.from,
+			editedRanges.qualifierValue.to
+		);
+
+		if (SKIP_QUALIFIERS.includes(qualifierKey.toLowerCase())) {
+			return []; // Keep query as is when editing year qualifiers
+		}
+
+		const baseClass = Object.entries(QUALIFIER_KEY_BY_BASE_CLASS).find(
+			([, key]) => key === qualifierKey
+		)?.[0];
+
+		if (baseClass) {
+			return [
+				['_qualifier', `"rdf:type":${baseClass} ${qualifierValue}`],
+				['min-reverseLinks.totalItems', '1'] // ensure results are linked/used atleast once
+			];
+		}
+
+		return [['_qualifier', qualifierKey + qualifierOperator + qualifierValue]];
+	}
+
+	/**
+	 * Otherwise keep query entries as is
+	 */
+	return [];
+}
+
+export default getEditedPartEntries;

lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.test.ts

@@ -0,0 +1,119 @@
+import { describe, it, expect } from 'vitest';
+import getEditedRanges from './getEditedRanges';
+
+describe('getEditedRanges', () => {
+	it('calculates the edited range for a simple free-text query', () => {
+		const query = 'hello';
+		const editedRanges = getEditedRanges(query, 5);
+		expect(editedRanges).toEqual({ from: 0, to: 5 });
+		expect(query.slice(editedRanges.from, editedRanges.to)).toBe('hello');
+	});
+
+	it('calculates the edited range (with included ranges of qualifier parts) when editing a qualifier', () => {
+		const query = 'hasTitle:"a"';
+		const editedRanges = getEditedRanges(query, 11);
+		expect(editedRanges).toEqual({
+			from: 0,
+			to: 12,
+			qualifierKey: {
+				from: 0,
+				to: 8
+			},
+			qualifierOperator: {
+				from: 8,
+				to: 9
+			},
+			qualifierValue: {
+				from: 9,
+				to: 12
+			}
+		});
+		expect(query.slice(editedRanges.from, editedRanges.to)).toBe('hasTitle:"a"');
+		expect(query.slice(editedRanges.qualifierKey?.from, editedRanges.qualifierKey?.to)).toBe(
+			'hasTitle'
+		);
+		expect(
+			query.slice(editedRanges.qualifierOperator?.from, editedRanges.qualifierOperator?.to)
+		).toBe(':');
+		expect(query.slice(editedRanges.qualifierValue?.from, editedRanges.qualifierValue?.to)).toBe(
+			'"a"'
+		);
+	});
+
+	it('calculates the edited range when editing a string', () => {
+		expect(getEditedRanges('"hello"', 6)).toEqual({
+			from: 0,
+			to: 7
+		});
+	});
+
+	it('calculates the edited range when editing a group', () => {
+		expect(getEditedRanges('(hello world)', 9)).toEqual({
+			from: 0,
+			to: 13
+		});
+	});
+
+	it('calculates the edited range when editing a part after a qualifier', () => {
+		expect(getEditedRanges('hasTitle:"a" hello', 18)).toEqual({
+			from: 12,
+			to: 18
+		});
+	});
+
+	it('calculates the edited range when editing a part after a qualifier', () => {
+		expect(getEditedRanges('hasTitle:"a" hello', 18)).toEqual({
+			from: 12,
+			to: 18
+		});
+	});
+
+	it('calculates the edited range when editing a part before a qualifier', () => {
+		expect(getEditedRanges('hello hasTitle:"a"', 5)).toEqual({
+			from: 0,
+			to: 6
+		});
+	});
+
+	it('calculates the edited range when editing a part surrounded by qualifiers', () => {
+		expect(getEditedRanges('hasTitle:"a" hello hasTitle:"b', 18)).toEqual({
+			from: 12,
+			to: 19
+		});
+	});
+
+	it('calculates the edited range when editing a part after a group', () => {
+		expect(getEditedRanges('(hi) hello', 7)).toEqual({
+			from: 4,
+			to: 10
+		});
+	});
+
+	it('calculates the edited range when editing a part before a group', () => {
+		expect(getEditedRanges('hello (hi)', 3)).toEqual({
+			from: 0,
+			to: 6
+		});
+	});
+
+	it('calculates the edited range when editing a part surrounded by groups', () => {
+		expect(getEditedRanges('(hi) hello (hej)', 7)).toEqual({
+			from: 4,
+			to: 11
+		});
+	});
+
+	it('calculates the edited range when editing a part before a boolean operator', () => {
+		expect(getEditedRanges('hello AND world', 5)).toEqual({
+			from: 0,
+			to: 6
+		});
+	});
+
+	it('calculates the edited range when editing a part after a boolean operator', () => {
+		expect(getEditedRanges('hello AND world', 12)).toEqual({
+			from: 9,
+			to: 15
+		});
+	});
+});

lxl-web/src/routes/api/[[lang=lang]]/supersearch/getEditedRanges.ts

@@ -0,0 +1,84 @@
+import { lxlQuery } from 'codemirror-lang-lxlquery';
+
+type Range = {
+	from: number;
+	to: number;
+};
+
+export type EditedRanges = Range & {
+	qualifierKey?: Range;
+	qualifierOperator?: Range;
+	qualifierValue?: Range;
+};
+
+function getEditedRanges(query: string, cursor: number): EditedRanges {
+	const tree = lxlQuery.language.parser.parse(query);
+	const innerNode = tree.resolveInner(cursor);
+
+	/**
+	 * Return `from` and `to` from qualifier parts if editing qualifier value
+	 */
+	if (innerNode.parent?.type.is('QualifierValue')) {
+		const qualifierNode = innerNode.parent.parent;
+		const qualifierKeyNode = qualifierNode?.getChild('QualifierKey');
+		const qualifierOperatorNode = qualifierNode?.getChild('QualifierOperator');
+		const qualiferValueNode = qualifierNode?.getChild('QualifierValue');
+		if (qualifierNode) {
+			return {
+				from: qualifierNode.from,
+				to: qualifierNode.to,
+				...(qualifierKeyNode && {
+					qualifierKey: { from: qualifierKeyNode.from, to: qualifierKeyNode.to }
+				}),
+				...(qualifierOperatorNode && {
+					qualifierOperator: { from: qualifierOperatorNode.from, to: qualifierOperatorNode.to }
+				}),
+				...(qualiferValueNode && {
+					qualifierValue: { from: qualiferValueNode.from, to: qualiferValueNode.to }
+				})
+			};
+		}
+	}
+
+	let from = 0;
+	let to = query.length;
+
+	/**
+	 * Adjust `from` and `to` if enclosed qualifiers or groups are found BEFORE the edited part
+	 * */
+	tree.iterate({
+		from: 0,
+		to: cursor,
+		enter(node) {
+			if (node.type.is('Qualifier') || node.type.is('Group') || node.type.is('BooleanOperator')) {
+				if (node.to > cursor) {
+					from = node.from;
+					to = node.to;
+				} else {
+					from = node.to;
+				}
+			}
+		}
+	});
+
+	/**
+	 * Adjust `from` and `to` if enclosed qualifiers or groups are found AFTER the edited part
+	 * */
+	tree.iterate({
+		from,
+		to,
+		enter(node) {
+			if (
+				(node.type.is('Qualifier') || node.type.is('Group') || node.type.is('BooleanOperator')) &&
+				node.from > cursor &&
+				node.from < to
+			) {
+				to = node.from;
+			}
+		}
+	});
+
+	return { from, to };
+}
+
+export default getEditedRanges;

packages/codemirror-lang-lxlquery/src/syntax.grammar

@@ -12,8 +12,8 @@ term {
 Group { "(" term* ")" } 
 
 BooleanQuery { 
-  (freetext | Qualifier | Group ) booleanOperator (freetext | Qualifier | Group ) 
-  (booleanOperator (freetext | Qualifier | Group ))+?
+  (freetext | Qualifier | Group ) BooleanOperator (freetext | Qualifier | Group ) 
+  (BooleanOperator (freetext | Qualifier | Group ))+?
 }
 
 Qualifier {
@@ -47,15 +47,15 @@ freetext {
 
   CompareOperator { ">" | "<" | ">=" | "<=" }
 
-  booleanOperator { "AND" | "OR" | "NOT" }
+  BooleanOperator { "AND" | "OR" | "NOT" }
 
   Wildcard { "*"+ }
 
   reserved { "includeEplikt" | "includePreliminary" }
 
   space { @whitespace+ }
 
-  @precedence { booleanOperator, reserved, Identifier }
+  @precedence { BooleanOperator, reserved, Identifier }
 }
 
 @detectDelim