From fb083906c395ad0cb91c1a0c308d8662d04784fe Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Wed, 28 Sep 2022 13:20:20 +0200 Subject: [PATCH 1/9] feat: Larger spacing between search panel middle --- app/configurator/components/select-dataset-step.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/configurator/components/select-dataset-step.tsx b/app/configurator/components/select-dataset-step.tsx index 99724502e..b3155975a 100644 --- a/app/configurator/components/select-dataset-step.tsx +++ b/app/configurator/components/select-dataset-step.tsx @@ -64,7 +64,7 @@ const useStyles = makeStyles(() => ({ }, panelMiddle: { paddingTop: 0, - paddingLeft: 6, + paddingLeft: 18, gridColumnStart: "middle", gridColumnEnd: "right", }, From 1374200aac37864d94171cae4c9790f116b09f39 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Wed, 28 Sep 2022 10:00:54 +0200 Subject: [PATCH 2/9] feat: Highlight search matches manually --- app/components/debug-search.tsx | 15 +++++++++++---- app/rdf/query-search-score-utils.spec.ts | 23 +++++++++++++++++++++++ app/rdf/query-search-score-utils.ts | 5 +++++ app/rdf/query-search.ts | 13 +++++++------ 4 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 app/rdf/query-search-score-utils.spec.ts diff --git a/app/components/debug-search.tsx b/app/components/debug-search.tsx index 45b7f7df6..35d33f713 100644 --- a/app/components/debug-search.tsx +++ b/app/components/debug-search.tsx @@ -127,10 +127,17 @@ const Search = ({ {cubes.data?.dataCubes.map((c) => { return (
- {c.highlightedTitle} - - {c?.highlightedDescription?.slice(0, 100) ?? "" + "..."} - + +
{c?.dataCube?.iri} diff --git a/app/rdf/query-search-score-utils.spec.ts b/app/rdf/query-search-score-utils.spec.ts new file mode 100644 index 000000000..1b1f445e2 --- /dev/null +++ b/app/rdf/query-search-score-utils.spec.ts @@ -0,0 +1,23 @@ +import { highlight } from "./query-search-score-utils"; + +describe("highlighting search words in qquery", () => { + it("should work", () => { + const tests = [ + ["Pollution is bad", "bad", "Pollution is bad"], + [ + "The assessment of bathing waters is made on the basis of hygienic quality using E.coli and intestina", + "Bathing", + "The assessment of bathing waters is made on the basis of hygienic quality using E.coli and intestina", + ], + [ + "GEB - Einmalvergütung für Photovoltaikanlagen", + "Einmalvergütung", + "GEB - Einmalvergütung für Photovoltaikanlagen", + ], + ] as [string, string, string][]; + for (const t of tests) { + const result = highlight(t[0], t[1]); + expect(result).toEqual(t[2]); + } + }); +}); diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index c9773b3b5..2deb96cbd 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -51,3 +51,8 @@ export const computeScores = ( } return infoPerCube; }; + +export const highlight = (text: string, query: string) => { + const re = new RegExp(query.toLowerCase().split(" ").join("|"), "gi"); + return text.replace(re, (m) => `${m}`); +}; diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index cb434bbf3..9e61fda97 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -8,11 +8,12 @@ import ParsingClient from "sparql-http-client/ParsingClient"; import { truthy } from "@/domain/types"; import { DataCubeSearchFilter } from "@/graphql/resolver-types"; +import { ResolvedDataCube } from "@/graphql/shared-types"; import * as ns from "@/rdf/namespace"; import { parseCube, parseIri, parseVersionHistory } from "@/rdf/parse"; import { fromStream } from "@/rdf/sparql-client"; -import { computeScores } from "./query-search-score-utils"; +import { computeScores, highlight } from "./query-search-score-utils"; const toNamedNode = (x: string) => { return `<${x}>`; @@ -290,7 +291,7 @@ export const searchCubes = async ({ // Sort the cubes per score using previously queries scores const results = cubes - .filter((c) => !!c?.data) + .filter((c): c is ResolvedDataCube => !!c?.data) .sort((a, b) => descending( infoPerCube[a?.data.iri!].score, @@ -299,10 +300,10 @@ export const searchCubes = async ({ ) .map((c) => ({ dataCube: c, - - // TODO Retrieve highlights - highlightedTitle: c!.data.title, - highlightedDescription: c!.data.description, + highlightedTitle: query ? highlight(c.data.title, query) : c.data.title, + highlightedDescription: query + ? highlight(c.data.description, query) + : c.data.description, })); return { From 5d171fddadb0e4547ea8121b49276b04dbf65df1 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Wed, 28 Sep 2022 10:18:20 +0200 Subject: [PATCH 3/9] refactor: Deduplicate helper function --- app/rdf/query-cube-metadata.ts | 15 +-------------- app/rdf/query-search.ts | 31 ++++++------------------------- app/rdf/query-utils.ts | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 39 deletions(-) create mode 100644 app/rdf/query-utils.ts diff --git a/app/rdf/query-cube-metadata.ts b/app/rdf/query-cube-metadata.ts index 4673ea6b0..5e421f042 100644 --- a/app/rdf/query-cube-metadata.ts +++ b/app/rdf/query-cube-metadata.ts @@ -7,6 +7,7 @@ import { schema, dcat, dcterms, cube } from "../../app/rdf/namespace"; import { DataCubeOrganization, DataCubeTheme } from "../graphql/query-hooks"; import { makeLocalesFilter } from "./query-labels"; +import { makeVisualizeDatasetFilter } from "./query-utils"; type RawDataCubeTheme = Omit; type RawDataCubeOrganization = Omit; @@ -129,20 +130,6 @@ export const queryDatasetCountByOrganization = async ({ .filter((r) => r.iri); }; -const makeVisualizeDatasetFilter = (options?: { includeDrafts?: boolean }) => { - const includeDrafts = options?.includeDrafts || false; - return sparql` - ?iri ${schema.workExample} . - ${ - includeDrafts - ? "" - : sparql`?iri ${schema.creativeWorkStatus} .` - } - FILTER NOT EXISTS {?iri ${schema.expires} ?expiryDate } - FILTER NOT EXISTS {?iri ${schema.validThrough} ?validThrough } - `; -}; - export const queryDatasetCountByTheme = async ({ sparqlClient, organization, diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 9e61fda97..7cd840284 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -1,4 +1,4 @@ -import { DESCRIBE, SELECT, sparql } from "@tpluscode/sparql-builder"; +import { DESCRIBE, SELECT } from "@tpluscode/sparql-builder"; import clownface from "clownface"; import { descending } from "d3"; import { Cube } from "rdf-cube-view-query"; @@ -14,6 +14,7 @@ import { parseCube, parseIri, parseVersionHistory } from "@/rdf/parse"; import { fromStream } from "@/rdf/sparql-client"; import { computeScores, highlight } from "./query-search-score-utils"; +import { makeVisualizeDatasetFilter } from "./query-utils"; const toNamedNode = (x: string) => { return `<${x}>`; @@ -59,29 +60,6 @@ const executeAndMeasure = async ( }; }; -const makeVisualizeFilter = (includeDrafts: boolean) => { - return sparql` - ?cube ${ns.schema.workExample} . - ?cube ${ns.schema.creativeWorkStatus} ?workStatus. - - ${ - !includeDrafts - ? ` - FILTER ( - ?workStatus IN () - )` - : "" - } - - FILTER ( - NOT EXISTS { ?cube ?validThrough . } - ) - FILTER ( - NOT EXISTS { ?cube ?expires . } - ) - `; -}; - const enhanceQuery = (rawQuery: string) => { const enhancedQuery = rawQuery .toLowerCase() @@ -150,7 +128,10 @@ export const searchCubes = async ({ ?versionHistory ${ns.schema.hasPart} ?cube. } - ${makeVisualizeFilter(!!includeDrafts)} + ${makeVisualizeDatasetFilter({ + includeDrafts: !!includeDrafts, + cubeIriVar: "?cube", + })} ${makeInFilter("about", aboutValues)} ${makeInFilter("theme", themeValues)} diff --git a/app/rdf/query-utils.ts b/app/rdf/query-utils.ts new file mode 100644 index 000000000..ee55370fc --- /dev/null +++ b/app/rdf/query-utils.ts @@ -0,0 +1,23 @@ +import { sparql } from "@tpluscode/sparql-builder"; + +import { schema } from "../../app/rdf/namespace"; + +export const makeVisualizeDatasetFilter = (options?: { + includeDrafts?: boolean; + cubeIriVar?: string; +}) => { + const cubeIriVar = options?.cubeIriVar || "?iri"; + const includeDrafts = options?.includeDrafts || false; + return sparql` + ${cubeIriVar} ${ + schema.workExample + } . + ${ + includeDrafts + ? "" + : sparql`${cubeIriVar} ${schema.creativeWorkStatus} .` + } + FILTER NOT EXISTS {${cubeIriVar} ${schema.expires} ?expiryDate } + FILTER NOT EXISTS {${cubeIriVar} ${schema.validThrough} ?validThrough } + `; +}; From b4b019e945d496880d0fec3ace19f8267ab6e74b Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Tue, 27 Sep 2022 23:36:53 +0200 Subject: [PATCH 4/9] fix: Can query on theme / publisher / creator --- app/rdf/query-search-score-utils.ts | 10 +- app/rdf/query-search.spec.ts | 5 +- app/rdf/query-search.ts | 139 ++++++++++++---------------- 3 files changed, 65 insertions(+), 89 deletions(-) diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index 2deb96cbd..7abc3491a 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -20,24 +20,24 @@ export const weights: Record = { */ export const computeScores = ( scoresRaw: any[], - { query }: { query?: string } + { query, identifierName }: { query?: string; identifierName: string } ) => { const infoPerCube = {} as Record; if (query) { for (let scoreRow of scoresRaw) { let score = 0; for (let [field, weight] of Object.entries(weights)) { - const val = scoreRow[field]?.value; + const val = scoreRow[field]; if (!val) { continue; } for (let tok of query.split(" ")) { - if (val.toLowerCase().includes(tok.toLowerCase())) { + if (val && val.toLowerCase().includes(tok.toLowerCase())) { score += weight; } } } - infoPerCube[scoreRow.cube.value] = { score }; + infoPerCube[scoreRow[identifierName]] = { score }; } for (let k of Object.keys(infoPerCube)) { if (infoPerCube[k]?.score === 0) { @@ -46,7 +46,7 @@ export const computeScores = ( } } else { for (let scoreRow of scoresRaw) { - infoPerCube[scoreRow.cube.value] = { score: 1 }; + infoPerCube[scoreRow[identifierName]] = { score: 1 }; } } return infoPerCube; diff --git a/app/rdf/query-search.spec.ts b/app/rdf/query-search.spec.ts index 299f0365c..3b30fc513 100644 --- a/app/rdf/query-search.spec.ts +++ b/app/rdf/query-search.spec.ts @@ -1,5 +1,3 @@ -import mapValues from "lodash/mapValues"; - import { computeScores, weights } from "./query-search-score-utils"; // jest.mock("rdf-ext", () => ({})); @@ -17,11 +15,12 @@ describe("compute scores", () => { { cube: "b", name: "national", description: "economy" }, { cube: "c", creatorLabel: "national" }, { cube: "d", creatorLabel: "" }, - ].map((x) => mapValues(x, (v) => ({ value: v }))); + ]; it("should compute weighted score per cube from score rows", () => { const reduced = computeScores(scores, { query: "national economy", + identifierName: "cube", }); expect(reduced["a"].score).toEqual(weights.name); expect(reduced["b"].score).toEqual(weights.name + weights.description); diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 7cd840284..e614f64f6 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -1,4 +1,5 @@ -import { DESCRIBE, SELECT } from "@tpluscode/sparql-builder"; +import { TemplateResult } from "@tpluscode/rdf-string/lib/TemplateResult"; +import { DESCRIBE, SELECT, sparql } from "@tpluscode/sparql-builder"; import clownface from "clownface"; import { descending } from "d3"; import { Cube } from "rdf-cube-view-query"; @@ -10,7 +11,7 @@ import { truthy } from "@/domain/types"; import { DataCubeSearchFilter } from "@/graphql/resolver-types"; import { ResolvedDataCube } from "@/graphql/shared-types"; import * as ns from "@/rdf/namespace"; -import { parseCube, parseIri, parseVersionHistory } from "@/rdf/parse"; +import { parseCube, parseIri } from "@/rdf/parse"; import { fromStream } from "@/rdf/sparql-client"; import { computeScores, highlight } from "./query-search-score-utils"; @@ -23,7 +24,7 @@ const makeInFilter = (varName: string, values: string[]) => { return ` ${ values.length > 0 - ? `FILTER ( + ? `FILTER (bound(?${varName}) && ?${varName} IN (${values.map(toNamedNode)}) )` : "" @@ -72,10 +73,17 @@ const enhanceQuery = (rawQuery: string) => { return enhancedQuery; }; -const contains = (left: string, right: string) => { +const icontains = (left: string, right: string) => { return `CONTAINS(LCASE(${left}), LCASE("${right}"))`; }; +type ResultRow = Record; +const parseResultRow = (row: ResultRow) => + Object.fromEntries(Object.entries(row).map(([k, v]) => [k, v.value])); + +const identity = (str: TemplateResult) => str; +const optional = (str: TemplateResult) => sparql`OPTIONAL { ${str} }`; + export const searchCubes = async ({ query: rawQuery, locale, @@ -107,15 +115,11 @@ export const searchCubes = async ({ filters?.filter((x) => x.type === "DataCubeAbout").map((v) => v.value) || []; - const scoresQuery = SELECT.DISTINCT`?cube ?versionHistory ?name ?description` + const scoresQuery = SELECT.DISTINCT`?cube ?versionHistory ?name ?description ?publisher ?themeName ?creatorLabel` .WHERE` ?cube a ${ns.cube.Cube}. ?cube ${ns.schema.name} ?name. - - ?cube ${ns.dcat.theme} ?theme. - ?cube ${ns.dcterms.creator} ?creator. - OPTIONAL { ?cube ${ns.schema.description} ?description. } @@ -127,6 +131,20 @@ export const searchCubes = async ({ OPTIONAL { ?versionHistory ${ns.schema.hasPart} ?cube. } + + OPTIONAL { ?cube ${ns.dcterms.publisher} ?publisher. } + + ${(themeValues.length > 0 ? identity : optional)(sparql` + ?cube ${ns.dcat.theme} ?theme. + ?theme ${ns.schema.name} ?themeName. + `)} + + ${(creatorValues.length > 0 ? identity : optional)( + sparql` + ?cube ${ns.dcterms.creator} ?creator. + ?creator ${ns.schema.name} ?creatorLabel. + ` + )} ${makeVisualizeDatasetFilter({ includeDrafts: !!includeDrafts, @@ -144,89 +162,43 @@ export const searchCubes = async ({ ?.split(" ") .slice(0, 1) .map( - (x) => `${contains("?name", x)} || ${contains("?description", x)}` + (x) => `${icontains("?name", x)} || ${icontains("?description", x)}` ) .join(" || ")} - + + || (bound(?publisher) && ${query + .split(" ") + .map((x) => icontains("?publisher", x)) + .join(" || ")}) + + || (bound(?themeName) && ${query + .split(" ") + .map((x) => icontains("?themeName", x)) + .join(" || ")}) + + || (bound(?creatorLabel) && ${query + .split(" ") + .map((x) => icontains("?creatorLabel", x)) + .join(" || ")}) + )` : "" } - `; - const scoresQuery2 = SELECT.DISTINCT`?cube ?versionHistory ?publisher ?themeName ?creatorLabel` - .WHERE` - ?cube a ${ns.cube.Cube}. - ?cube ${ns.schema.name} ?name. - - ?cube ${ns.dcat.theme} ?theme. - ?cube ${ns.dcterms.creator} ?creator. - - OPTIONAL { - ?cube ${ns.schema.about} ?about. - } - - OPTIONAL { - ?versionHistory ${ns.schema.hasPart} ?cube. - } - - ${makeVisualizeFilter(!!includeDrafts)} - - ${makeInFilter("about", aboutValues)} - ${makeInFilter("theme", themeValues)} - ${makeInFilter("creator", creatorValues)} - - ${ - query && query.length > 0 - ? sparql` - - OPTIONAL { - ?cube ${ns.dcterms.publisher} ?publisher. - FILTER(${query - .split(" ") - .map((x) => contains("?publisher", x)) - .join(" || ")}) . - } - - OPTIONAL { - ?theme ${ns.schema.name} ?themeName. - FILTER(${query - .split(" ") - .map((x) => contains("?themeName", x)) - .join(" || ")}) . - } - - - OPTIONAL { - - ?creator ${ns.schema.name} ?creatorLabel. - FILTER(${query - .split(" ") - .map((x) => contains("?creatorLabel", x)) - .join(" || ")}) . - } - ` - : "" - } - - `; - - let scoreResults = await executeAndMeasure(sparqlClient, scoresQuery); + const scoreResults = await executeAndMeasure(sparqlClient, scoresQuery); queries.push({ ...scoreResults.meta, label: "scores1", }); - if (scoreResults.data.length === 0) { - scoreResults = await executeAndMeasure(sparqlClient, scoresQuery2); - queries.push({ - ...scoreResults.meta, - label: "scores2", - }); - } - - const infoPerCube = computeScores(scoreResults.data, { + const data = scoreResults.data.map((x) => parseResultRow(x as ResultRow)); + const versionHistoryPerCube = Object.fromEntries( + data.map((d) => [d.cube, d.versionHistory]) + ); + const infoPerCube = computeScores(data, { query: query, + identifierName: "cube", }); // Find information on cubes @@ -234,7 +206,12 @@ export const searchCubes = async ({ // under the maximum score and only retrieve those cubes // The query could also dedup directly the version of the cubes const cubeIris = Object.keys(infoPerCube); - const cubesQuery = DESCRIBE`${cubeIris.map((x) => `<${x}>`).join(" ")}`; + + const sortedCubeIris = cubeIris.sort((a, b) => + descending(infoPerCube[a].score, infoPerCube[b].score) + ); + + const cubesQuery = DESCRIBE`${sortedCubeIris.map((x) => `<${x}>`).join(" ")}`; if (!locale) { throw new Error("Must pass locale"); @@ -260,7 +237,7 @@ export const searchCubes = async ({ .map((cubeNode) => { const cube = cubeNode as unknown as Cube; const iri = parseIri(cube); - const versionHistory = parseVersionHistory(cube); + const versionHistory = versionHistoryPerCube[iri]; const dedupIdentifier = versionHistory || iri; if (seen.has(dedupIdentifier)) { return null; From 8fe93d9bf4b3f909733627aba827c44e03e801c0 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Wed, 28 Sep 2022 15:16:54 +0200 Subject: [PATCH 5/9] refactor: Shorten cypress selectors --- cypress/integration/filters.spec.ts | 2 +- cypress/integration/search.spec.ts | 10 ++++++++-- cypress/integration/selectors.ts | 15 ++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cypress/integration/filters.spec.ts b/cypress/integration/filters.spec.ts index 545a4755a..33a629dcc 100644 --- a/cypress/integration/filters.spec.ts +++ b/cypress/integration/filters.spec.ts @@ -15,7 +15,7 @@ describe("Filters", () => { ); waitForChartToBeLoaded(); - selectors.edition.findConfiguratorFilters(cy).within(() => { + selectors.edition.configFilters(cy).within(() => { cy.findByText("1. production region"); cy.findByText("2. stand structure"); cy.findByText("3. evaluation type"); diff --git a/cypress/integration/search.spec.ts b/cypress/integration/search.spec.ts index 9fb0b5306..c46bcfa97 100644 --- a/cypress/integration/search.spec.ts +++ b/cypress/integration/search.spec.ts @@ -1,3 +1,5 @@ +import selectors from "./selectors"; + Cypress.on("uncaught:exception", (err) => { if (err.message.includes("> ResizeObserver loop")) { return false; @@ -11,7 +13,11 @@ describe("Searching for charts", () => { ); cy.waitForNetworkIdle(1000); - cy.get("#datasetSearch").should("have.attr", "value", "category"); - cy.get("#dataset-include-drafts").should("have.attr", "checked"); + selectors.search.searchInput(cy).should("have.attr", "value", "category"); + selectors.search.draftsCheckbox(cy).should("have.attr", "checked"); + }); + + it("should have coherent numbers between the side panel and the filtered results", () => { + cy.visit(`/en/browse`); }); }); diff --git a/cypress/integration/selectors.ts b/cypress/integration/selectors.ts index bf3b56ff8..8fe60df0e 100644 --- a/cypress/integration/selectors.ts +++ b/cypress/integration/selectors.ts @@ -2,15 +2,16 @@ type Cy = Cypress.Chainable; const selectors = { search: { - findSearchInput: (cy: Cy) => cy.get("#datasetSearch"), - findIncludeDraftsCheckbox: (cy: Cy) => cy.get("#dataset-include-drafts"), + searchInput: (cy: Cy) => cy.get("#datasetSearch"), + draftsCheckbox: (cy: Cy) => cy.get("#dataset-include-drafts"), + }, + panels: { + left: (cy: Cy) => cy.get('[data-name="panel-left"]'), + right: (cy: Cy) => cy.get('[data-name="panel-right"]'), }, edition: { - findLeftPanel: (cy: Cy) => cy.get('[data-name="panel-left"]'), - findRightPanel: (cy: Cy) => cy.get('[data-name="panel-right"]'), - findConfiguratorFilters: (cy: Cy) => - cy.findByTestId("configurator-filters"), - findChartFiltersList: (cy: Cy) => cy.findByTestId("chart-filters-list"), + configFilters: (cy: Cy) => cy.findByTestId("configurator-filters"), + chartFilters: (cy: Cy) => cy.findByTestId("chart-filters-list"), }, }; From 81cd877a318d802c657010d6e07aa8864ebb4e92 Mon Sep 17 00:00:00 2001 From: Patrick Browne Date: Thu, 29 Sep 2022 10:36:28 +0200 Subject: [PATCH 6/9] fix: Margin right of publish button --- app/configurator/components/stepper.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/configurator/components/stepper.tsx b/app/configurator/components/stepper.tsx index 16e800730..1131cc890 100644 --- a/app/configurator/components/stepper.tsx +++ b/app/configurator/components/stepper.tsx @@ -116,7 +116,7 @@ export const StepperDumb = ({ stepState={steps[currentStepIndex] as StepState | undefined} /> - +