From c7d29513a9eca1a7f784b16758f3c9c5a3b4d747 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 21 Jun 2024 13:43:15 +0200 Subject: [PATCH 1/8] perf: Optimize all termsets fetching --- app/rdf/query-termsets.ts | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/app/rdf/query-termsets.ts b/app/rdf/query-termsets.ts index 30b1a30ff..14f767f3d 100644 --- a/app/rdf/query-termsets.ts +++ b/app/rdf/query-termsets.ts @@ -15,30 +15,22 @@ export const queryAllTermsets = async (options: { const qs = await sparqlClient.query.select( `PREFIX cube: PREFIX meta: -PREFIX rdf: PREFIX schema: -PREFIX sh: -SELECT DISTINCT (COUNT(distinct ?iri) as ?count) ?termsetIri ?termsetLabel WHERE { - ?iri cube:observationConstraint/sh:property ?dimension . - ?dimension a cube:KeyDimension . - ?dimension sh:in/rdf:rest*/rdf:first ?value. - ?value schema:inDefinedTermSet ?termsetIri . - ?termsetIri a meta:SharedDimension . - ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} +SELECT DISTINCT (COUNT(DISTINCT ?cubeIri) as ?count) ?termsetIri ?termsetLabel WHERE { + ?termsetIri meta:isUsedIn ?cubeIri . + ${makeVisualizeDatasetFilter({ + includeDrafts: !!includeDrafts, + cubeIriVar: "?cubeIri", + })} + ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} - ${makeVisualizeDatasetFilter({ - includeDrafts: !!includeDrafts, - cubeIriVar: "?iri", - }).toString()} - - } GROUP BY ?termsetIri ?termsetLabel`, { - operation: "postUrlencoded", - } +} GROUP BY ?termsetIri ?termsetLabel`, + { operation: "postUrlencoded" } ); return qs.map((result) => ({ - count: Number(result.count.value), + count: +result.count.value, termset: { __typename: "Termset", iri: result.termsetIri.value, From 82ed3b1b87d955f843fb017f5aaa3e2018e41b90 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 21 Jun 2024 13:46:59 +0200 Subject: [PATCH 2/8] fix: Filtering by termsets --- app/rdf/query-search.ts | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 627b70656..ba0613fc6 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -214,7 +214,6 @@ const mkScoresQuery = ( sh:path ?dimensionIri ; cubeMeta:dataKind/time:unitType ?unitType ; cubeMeta:dataKind/time:unitType <${unitNode}>. - ${buildLocalizedSubQuery( "dimension", "schema:name", @@ -229,20 +228,7 @@ const mkScoresQuery = ( return ` VALUES (?termsetIri) {${sharedDimensions.map((sd) => `(<${sd}>)`).join(" ")}} ?iri a cube:Cube . - ?iri cube:observationConstraint/sh:property ?dimension . - ?dimension a cube:KeyDimension . - ?dimension sh:in/rdf:first ?value. - ?dimension sh:path ?dimensionIri . - ${buildLocalizedSubQuery( - "dimension", - "schema:name", - "dimensionLabel", - { - locale, - } - )} - - ?value schema:inDefinedTermSet ?termsetIri . + ?termsetIri meta:isUsedIn ?iri . ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })}`; } }) From 4fc46a6edc93e5461dcf4594fa345adb53f17fa0 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 21 Jun 2024 13:50:51 +0200 Subject: [PATCH 3/8] fix: If item is selected, expand the menu --- app/browser/dataset-browse.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index f8dedcb5f..bb6ac90b4 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -611,8 +611,7 @@ const NavSection = ({ (item) => item.label ); }, [counts, items]); - const { isOpen, open, close } = useDisclosure(); - + const { isOpen, open, close } = useDisclosure(!!currentFilter); return (
From 7d170a6db0979c1d1506c41a73debf4d7d2695f4 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 21 Jun 2024 13:52:57 +0200 Subject: [PATCH 4/8] refactor: Have one, not two files for querying termsets --- app/rdf/light-cube.ts | 2 +- app/rdf/query-cube-termsets.ts | 91 ---------------------------------- app/rdf/query-termsets.ts | 89 ++++++++++++++++++++++++++++++++- 3 files changed, 89 insertions(+), 93 deletions(-) delete mode 100644 app/rdf/query-cube-termsets.ts diff --git a/app/rdf/light-cube.ts b/app/rdf/light-cube.ts index 9a279012f..1f32f4f98 100644 --- a/app/rdf/light-cube.ts +++ b/app/rdf/light-cube.ts @@ -10,7 +10,7 @@ import { getCubePreview } from "@/rdf/query-cube-preview"; import { getCubeComponentTermsets, getCubeTermsets, -} from "@/rdf/query-cube-termsets"; +} from "@/rdf/query-termsets"; type LightCubeOptions = { iri: string; diff --git a/app/rdf/query-cube-termsets.ts b/app/rdf/query-cube-termsets.ts deleted file mode 100644 index 4f5f21dd0..000000000 --- a/app/rdf/query-cube-termsets.ts +++ /dev/null @@ -1,91 +0,0 @@ -import groupBy from "lodash/groupBy"; -import ParsingClient from "sparql-http-client/ParsingClient"; - -import { ComponentTermsets, Termset } from "@/domain/data"; -import { buildLocalizedSubQuery } from "@/rdf/query-utils"; - -export const getCubeTermsets = async ( - iri: string, - options: { - locale: string; - sparqlClient: ParsingClient; - } -): Promise => { - const { sparqlClient, locale } = options; - const qs = await sparqlClient.query.select( - `PREFIX cube: -PREFIX meta: -PREFIX rdf: -PREFIX schema: -PREFIX sh: - -SELECT DISTINCT ?termsetIri ?termsetLabel WHERE { - VALUES (?iri) {(<${iri}>)} - - ?iri cube:observationConstraint/sh:property ?dimension . - ?dimension a cube:KeyDimension . - ?dimension sh:in/rdf:rest*/rdf:first ?value. - ?value schema:inDefinedTermSet ?termsetIri . - ?termsetIri a meta:SharedDimension . - - ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} - -}` - ); - - return qs.map((result) => ({ - iri: result.termsetIri.value, - label: result.termsetLabel.value, - __typename: "Termset", - })); -}; - -export const getCubeComponentTermsets = async ( - iri: string, - options: { - locale: string; - sparqlClient: ParsingClient; - } -): Promise => { - const { sparqlClient, locale } = options; - const query = `PREFIX cube: -PREFIX meta: -PREFIX rdf: -PREFIX schema: -PREFIX sh: - -SELECT DISTINCT ?dimensionIri ?dimensionLabel ?termsetIri ?termsetLabel WHERE { - VALUES (?iri) {(<${iri}>)} - - ?iri cube:observationConstraint/sh:property ?dimension . - ?dimension sh:path ?dimensionIri . - ?dimension a cube:KeyDimension . - ?dimension sh:in/rdf:rest*/rdf:first ?value. - ?value schema:inDefinedTermSet ?termsetIri . - ?termsetIri a meta:SharedDimension . - - ${buildLocalizedSubQuery("dimension", "schema:name", "dimensionLabel", { locale })} - ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} - -}`; - const qs = await sparqlClient.query.select(query); - - const parsed = qs.map((result) => ({ - dimensionIri: result.dimensionIri.value, - dimensionLabel: result.dimensionLabel.value, - iri: result.termsetIri.value, - label: result.termsetLabel.value, - })); - - const grouped = Object.entries(groupBy(parsed, (r) => r.dimensionIri)); - - return grouped.map(([dimensionIri, termsets]) => ({ - iri: dimensionIri, - label: termsets[0].dimensionLabel, - termsets: termsets.map(({ iri, label }) => ({ - iri, - label, - __typename: "Termset", - })), - })); -}; diff --git a/app/rdf/query-termsets.ts b/app/rdf/query-termsets.ts index 14f767f3d..2e7089790 100644 --- a/app/rdf/query-termsets.ts +++ b/app/rdf/query-termsets.ts @@ -1,6 +1,7 @@ +import groupBy from "lodash/groupBy"; import ParsingClient from "sparql-http-client/ParsingClient"; -import { Termset } from "@/domain/data"; +import { ComponentTermsets, Termset } from "@/domain/data"; import { buildLocalizedSubQuery, makeVisualizeDatasetFilter, @@ -38,3 +39,89 @@ SELECT DISTINCT (COUNT(DISTINCT ?cubeIri) as ?count) ?termsetIri ?termsetLabel W }, })); }; + +export const getCubeTermsets = async ( + iri: string, + options: { + locale: string; + sparqlClient: ParsingClient; + } +): Promise => { + const { sparqlClient, locale } = options; + const qs = await sparqlClient.query.select( + `PREFIX cube: +PREFIX meta: +PREFIX rdf: +PREFIX schema: +PREFIX sh: + +SELECT DISTINCT ?termsetIri ?termsetLabel WHERE { + VALUES (?iri) {(<${iri}>)} + + ?iri cube:observationConstraint/sh:property ?dimension . + ?dimension a cube:KeyDimension . + ?dimension sh:in/rdf:rest*/rdf:first ?value. + ?value schema:inDefinedTermSet ?termsetIri . + ?termsetIri a meta:SharedDimension . + + ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} + +}` + ); + + return qs.map((result) => ({ + iri: result.termsetIri.value, + label: result.termsetLabel.value, + __typename: "Termset", + })); +}; + +export const getCubeComponentTermsets = async ( + iri: string, + options: { + locale: string; + sparqlClient: ParsingClient; + } +): Promise => { + const { sparqlClient, locale } = options; + const query = `PREFIX cube: +PREFIX meta: +PREFIX rdf: +PREFIX schema: +PREFIX sh: + +SELECT DISTINCT ?dimensionIri ?dimensionLabel ?termsetIri ?termsetLabel WHERE { + VALUES (?iri) {(<${iri}>)} + + ?iri cube:observationConstraint/sh:property ?dimension . + ?dimension sh:path ?dimensionIri . + ?dimension a cube:KeyDimension . + ?dimension sh:in/rdf:rest*/rdf:first ?value. + ?value schema:inDefinedTermSet ?termsetIri . + ?termsetIri a meta:SharedDimension . + + ${buildLocalizedSubQuery("dimension", "schema:name", "dimensionLabel", { locale })} + ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} + +}`; + const qs = await sparqlClient.query.select(query); + + const parsed = qs.map((result) => ({ + dimensionIri: result.dimensionIri.value, + dimensionLabel: result.dimensionLabel.value, + iri: result.termsetIri.value, + label: result.termsetLabel.value, + })); + + const grouped = Object.entries(groupBy(parsed, (r) => r.dimensionIri)); + + return grouped.map(([dimensionIri, termsets]) => ({ + iri: dimensionIri, + label: termsets[0].dimensionLabel, + termsets: termsets.map(({ iri, label }) => ({ + iri, + label, + __typename: "Termset", + })), + })); +}; From 467748f3f84fd6e18def664a051a1cc268e25e93 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 21 Jun 2024 14:01:09 +0200 Subject: [PATCH 5/8] perf: Optimize cube termsets fetching --- app/rdf/query-termsets.ts | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/app/rdf/query-termsets.ts b/app/rdf/query-termsets.ts index 2e7089790..dd07fad39 100644 --- a/app/rdf/query-termsets.ts +++ b/app/rdf/query-termsets.ts @@ -49,24 +49,15 @@ export const getCubeTermsets = async ( ): Promise => { const { sparqlClient, locale } = options; const qs = await sparqlClient.query.select( - `PREFIX cube: -PREFIX meta: -PREFIX rdf: + `PREFIX meta: PREFIX schema: -PREFIX sh: SELECT DISTINCT ?termsetIri ?termsetLabel WHERE { - VALUES (?iri) {(<${iri}>)} - - ?iri cube:observationConstraint/sh:property ?dimension . - ?dimension a cube:KeyDimension . - ?dimension sh:in/rdf:rest*/rdf:first ?value. - ?value schema:inDefinedTermSet ?termsetIri . - ?termsetIri a meta:SharedDimension . - + VALUES (?cubeIri) {(<${iri}>)} + ?termsetIri meta:isUsedIn ?cubeIri . ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} - -}` +}`, + { operation: "postUrlencoded" } ); return qs.map((result) => ({ From ddb1d4850ef34d52853346d302152e0367871803 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Wed, 3 Jul 2024 10:25:14 +0200 Subject: [PATCH 6/8] refactor: Clean up --- app/rdf/query-termsets.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/app/rdf/query-termsets.ts b/app/rdf/query-termsets.ts index dd07fad39..e750a3648 100644 --- a/app/rdf/query-termsets.ts +++ b/app/rdf/query-termsets.ts @@ -25,7 +25,6 @@ SELECT DISTINCT (COUNT(DISTINCT ?cubeIri) as ?count) ?termsetIri ?termsetLabel W cubeIriVar: "?cubeIri", })} ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} - } GROUP BY ?termsetIri ?termsetLabel`, { operation: "postUrlencoded" } ); @@ -55,7 +54,7 @@ PREFIX schema: SELECT DISTINCT ?termsetIri ?termsetLabel WHERE { VALUES (?cubeIri) {(<${iri}>)} ?termsetIri meta:isUsedIn ?cubeIri . - ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} + ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} }`, { operation: "postUrlencoded" } ); @@ -83,19 +82,19 @@ PREFIX sh: SELECT DISTINCT ?dimensionIri ?dimensionLabel ?termsetIri ?termsetLabel WHERE { VALUES (?iri) {(<${iri}>)} - ?iri cube:observationConstraint/sh:property ?dimension . - ?dimension sh:path ?dimensionIri . - ?dimension a cube:KeyDimension . - ?dimension sh:in/rdf:rest*/rdf:first ?value. + ?dimension + sh:path ?dimensionIri ; + a cube:KeyDimension ; + sh:in/rdf:rest*/rdf:first ?value . ?value schema:inDefinedTermSet ?termsetIri . ?termsetIri a meta:SharedDimension . - ${buildLocalizedSubQuery("dimension", "schema:name", "dimensionLabel", { locale })} ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} - }`; - const qs = await sparqlClient.query.select(query); + const qs = await sparqlClient.query.select(query, { + operation: "postUrlencoded", + }); const parsed = qs.map((result) => ({ dimensionIri: result.dimensionIri.value, From 9bfb295d2869281d267e1877d64600349ddcf73c Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Wed, 28 Aug 2024 11:37:28 +0200 Subject: [PATCH 7/8] perf: Optimize fetching of component termsets ...it looks like using VALUES is making the query much slower. As here we always only use one cube iri, we can specify it directly and speed up things significantly. --- app/rdf/query-termsets.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/rdf/query-termsets.ts b/app/rdf/query-termsets.ts index e750a3648..86a207d1b 100644 --- a/app/rdf/query-termsets.ts +++ b/app/rdf/query-termsets.ts @@ -81,8 +81,7 @@ PREFIX schema: PREFIX sh: SELECT DISTINCT ?dimensionIri ?dimensionLabel ?termsetIri ?termsetLabel WHERE { - VALUES (?iri) {(<${iri}>)} - ?iri cube:observationConstraint/sh:property ?dimension . + <${iri}> cube:observationConstraint/sh:property ?dimension . ?dimension sh:path ?dimensionIri ; a cube:KeyDimension ; From 3de64f1fcd3313e60e4883006e3c66ea727bd447 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Wed, 28 Aug 2024 11:47:06 +0200 Subject: [PATCH 8/8] perf: Optimize getCubeTermsets --- app/rdf/query-termsets.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/rdf/query-termsets.ts b/app/rdf/query-termsets.ts index 86a207d1b..1798c5478 100644 --- a/app/rdf/query-termsets.ts +++ b/app/rdf/query-termsets.ts @@ -52,8 +52,7 @@ export const getCubeTermsets = async ( PREFIX schema: SELECT DISTINCT ?termsetIri ?termsetLabel WHERE { - VALUES (?cubeIri) {(<${iri}>)} - ?termsetIri meta:isUsedIn ?cubeIri . + ?termsetIri meta:isUsedIn <${iri}> . ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })} }`, { operation: "postUrlencoded" }