Merge pull request #1265 from visualize-admin/perf/optimize-dataset-p…

…review-performance

perf: Optimize dataset previews performance

CHANGELOG.md

@@ -17,6 +17,7 @@ You can also check the [release page](https://github.com/visualize-admin/visuali
   - Hierarchy names are now correctly retrieved
 - Performance
   - We no longer fetch shape when initalizing the cube, as we might need to re-fetch it again if a newer cube is required
+  - Vastly improved performance of dataset preview by using a new version of `cube-view-query` library (`View.preview`)
 
 # [3.24.0] - 2023-11-08
 

app/rdf/queries.ts

@@ -842,6 +842,8 @@ const buildFilters = ({
   });
 };
 
+type ObservationRaw = Record<string, Literal | NamedNode>;
+
 async function fetchViewObservations({
   limit,
   observationsView,
@@ -851,37 +853,28 @@ async function fetchViewObservations({
   observationsView: View;
   disableDistinct: boolean;
 }) {
-  /**
-   * Add LIMIT to query
-   */
-  if (limit !== undefined) {
-    // From https://github.com/zazuko/cube-creator/blob/a32a90ff93b2c6c1c5ab8fd110a9032a8d179670/apis/core/lib/domain/observations/lib/index.ts#L41
-    observationsView.ptr.addOut(ns.cubeView.projection, (projection: $FixMe) =>
-      projection.addOut(ns.cubeView.limit, limit)
-    );
-  }
-
-  const queryOptions = {
-    disableDistinct,
-  };
+  const fullQuery = observationsView.observationsQuery({ disableDistinct });
+  const query = (
+    limit ? fullQuery.previewQuery({ limit }) : fullQuery.query
+  ).toString();
 
-  const query = observationsView
-    .observationsQuery(queryOptions)
-    .query.toString();
+  let observationsRaw: PromiseValue<ObservationRaw[]> | undefined;
 
-  let observationsRaw:
-    | PromiseValue<ReturnType<typeof observationsView.observations>>
-    | undefined;
   try {
-    observationsRaw = await observationsView.observations(queryOptions);
+    observationsRaw = await (limit
+      ? observationsView.preview({ limit })
+      : observationsView.observations({ disableDistinct }));
   } catch (e) {
-    console.warn("Query failed", query);
+    console.warn("Observations query failed!", query);
     throw new Error(
       `Could not retrieve data: ${e instanceof Error ? e.message : e}`
     );
   }
 
-  return { query, observationsRaw };
+  return {
+    query,
+    observationsRaw,
+  };
 }
 
 type RDFObservation = Record<string, Literal | NamedNode<string>>;
@@ -892,7 +885,8 @@ function parseObservation(
 ): (value: RDFObservation) => Observation {
   return (obs) => {
     const res = {} as Observation;
-    for (let d of cubeDimensions) {
+
+    for (const d of cubeDimensions) {
       const label = obs[labelDimensionIri(d.data.iri)]?.value;
       const termType = obs[d.data.iri]?.termType;
 

app/typings/rdf.d.ts

@@ -101,13 +101,18 @@ declare module "rdf-cube-view-query" {
     dimension(options: { cubeDimension: NamedNode | string }): Dimension | null;
     observationsQuery({ disableDistinct }: { disableDistinct?: boolean }): {
       query: $FixMe;
+      previewQuery: $FixMe;
       dimensionMap: Map;
     };
     async observations({
       disableDistinct,
     }: {
       disableDistinct?: boolean;
     }): Promise<Record<string, Literal | NamedNode>[]>;
+    async preview(options: {
+      limit?: number;
+      offset?: number;
+    }): Promise<Record<string, Literal | NamedNode>[]>;
     addDimension(dimension: Dimension): View;
     createDimension(options: $FixMe): Dimension;
     setDefaultColumns(): void;