Merge pull request #1633 from visualize-admin/feat/new-termset-triples

perf: Optimize termsets fetching
visualize-admin · Aug 28, 2024 · 4161581 · 4161581
2 parents 115d54b + 3de64f1
commit 4161581
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 129 deletions.
diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx
@@ -626,8 +626,7 @@ const NavSection = ({
       (item) => item.label
     );
   }, [counts, items]);
-  const { isOpen, open, close } = useDisclosure();
-
+  const { isOpen, open, close } = useDisclosure(!!currentFilter);
   return (
     <div>
       <NavSectionTitle theme={theme} sx={{ mb: "block" }}>

diff --git a/app/rdf/light-cube.ts b/app/rdf/light-cube.ts
@@ -10,7 +10,7 @@ import { getCubePreview } from "@/rdf/query-cube-preview";
 import {
   getCubeComponentTermsets,
   getCubeTermsets,
-} from "@/rdf/query-cube-termsets";
+} from "@/rdf/query-termsets";
 
 type LightCubeOptions = {
   iri: string;

diff --git a/app/rdf/query-cube-termsets.ts b/app/rdf/query-cube-termsets.ts
diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts
@@ -213,7 +213,6 @@ const mkScoresQuery = (
               sh:path ?dimensionIri ;
               cubeMeta:dataKind/time:unitType ?unitType ;
               cubeMeta:dataKind/time:unitType <${unitNode}>.
-
             ${buildLocalizedSubQuery(
               "dimension",
               "schema:name",
@@ -228,20 +227,7 @@ const mkScoresQuery = (
             return `
             VALUES (?termsetIri) {${sharedDimensions.map((sd) => `(<${sd}>)`).join(" ")}}
             ?iri a cube:Cube .
-            ?iri cube:observationConstraint/sh:property ?dimension .
-            ?dimension a cube:KeyDimension .
-            ?dimension sh:in/rdf:first ?value.
-            ?dimension sh:path ?dimensionIri .
-            ${buildLocalizedSubQuery(
-              "dimension",
-              "schema:name",
-              "dimensionLabel",
-              {
-                locale,
-              }
-            )}
-      
-            ?value schema:inDefinedTermSet ?termsetIri .
+            ?termsetIri meta:isUsedIn ?iri .
             ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })}`;
           }
         })

diff --git a/app/rdf/query-termsets.ts b/app/rdf/query-termsets.ts
@@ -1,6 +1,7 @@
+import groupBy from "lodash/groupBy";
 import ParsingClient from "sparql-http-client/ParsingClient";
 
-import { Termset } from "@/domain/data";
+import { ComponentTermsets, Termset } from "@/domain/data";
 import {
   buildLocalizedSubQuery,
   makeVisualizeDatasetFilter,
@@ -15,34 +16,100 @@ export const queryAllTermsets = async (options: {
   const qs = await sparqlClient.query.select(
     `PREFIX cube: <https://cube.link/>
 PREFIX meta: <https://cube.link/meta/>
-PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
 PREFIX schema: <http://schema.org/>
-PREFIX sh: <http://www.w3.org/ns/shacl#>
-
-SELECT DISTINCT (COUNT(distinct ?iri) as ?count) ?termsetIri ?termsetLabel WHERE {
-    ?iri cube:observationConstraint/sh:property ?dimension .
-    ?dimension a cube:KeyDimension .
-    ?dimension sh:in/rdf:rest*/rdf:first ?value.
-    ?value schema:inDefinedTermSet ?termsetIri .
-    ?termsetIri a meta:SharedDimension .
-    ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })}
-
-    ${makeVisualizeDatasetFilter({
-      includeDrafts: !!includeDrafts,
-      cubeIriVar: "?iri",
-    }).toString()}
 
-  }      GROUP BY ?termsetIri ?termsetLabel`, {
-    operation: "postUrlencoded",
-  }
+SELECT DISTINCT (COUNT(DISTINCT ?cubeIri) as ?count) ?termsetIri ?termsetLabel WHERE {
+  ?termsetIri meta:isUsedIn ?cubeIri .
+  ${makeVisualizeDatasetFilter({
+    includeDrafts: !!includeDrafts,
+    cubeIriVar: "?cubeIri",
+  })}
+  ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })}
+} GROUP BY ?termsetIri ?termsetLabel`,
+    { operation: "postUrlencoded" }
   );
 
   return qs.map((result) => ({
-    count: Number(result.count.value),
+    count: +result.count.value,
     termset: {
       __typename: "Termset",
       iri: result.termsetIri.value,
       label: result.termsetLabel.value,
     },
   }));
 };
+
+export const getCubeTermsets = async (
+  iri: string,
+  options: {
+    locale: string;
+    sparqlClient: ParsingClient;
+  }
+): Promise<Termset[]> => {
+  const { sparqlClient, locale } = options;
+  const qs = await sparqlClient.query.select(
+    `PREFIX meta: <https://cube.link/meta/>
+PREFIX schema: <http://schema.org/>
+
+SELECT DISTINCT ?termsetIri ?termsetLabel WHERE {
+  ?termsetIri meta:isUsedIn <${iri}> .
+  ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })}
+}`,
+    { operation: "postUrlencoded" }
+  );
+
+  return qs.map((result) => ({
+    iri: result.termsetIri.value,
+    label: result.termsetLabel.value,
+    __typename: "Termset",
+  }));
+};
+
+export const getCubeComponentTermsets = async (
+  iri: string,
+  options: {
+    locale: string;
+    sparqlClient: ParsingClient;
+  }
+): Promise<ComponentTermsets[]> => {
+  const { sparqlClient, locale } = options;
+  const query = `PREFIX cube: <https://cube.link/>
+PREFIX meta: <https://cube.link/meta/>
+PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
+PREFIX schema: <http://schema.org/>
+PREFIX sh: <http://www.w3.org/ns/shacl#>
+
+SELECT DISTINCT ?dimensionIri ?dimensionLabel ?termsetIri ?termsetLabel WHERE {
+  <${iri}> cube:observationConstraint/sh:property ?dimension .
+  ?dimension
+    sh:path ?dimensionIri ;
+    a cube:KeyDimension ;
+    sh:in/rdf:rest*/rdf:first ?value .
+  ?value schema:inDefinedTermSet ?termsetIri .
+  ?termsetIri a meta:SharedDimension .
+  ${buildLocalizedSubQuery("dimension", "schema:name", "dimensionLabel", { locale })}
+  ${buildLocalizedSubQuery("termsetIri", "schema:name", "termsetLabel", { locale })}
+}`;
+  const qs = await sparqlClient.query.select(query, {
+    operation: "postUrlencoded",
+  });
+
+  const parsed = qs.map((result) => ({
+    dimensionIri: result.dimensionIri.value,
+    dimensionLabel: result.dimensionLabel.value,
+    iri: result.termsetIri.value,
+    label: result.termsetLabel.value,
+  }));
+
+  const grouped = Object.entries(groupBy(parsed, (r) => r.dimensionIri));
+
+  return grouped.map(([dimensionIri, termsets]) => ({
+    iri: dimensionIri,
+    label: termsets[0].dimensionLabel,
+    termsets: termsets.map(({ iri, label }) => ({
+      iri,
+      label,
+      __typename: "Termset",
+    })),
+  }));
+};