perf: Only load necessary components for observations query

...as it takes significant amount of time to use getCubeDimensions in cases of bigger datasets, we should only load necessary components. This also means that we only show what's defined in chart in the preview table and data download.
visualize-admin · May 10, 2023 · 0df4747 · 0df4747
1 parent 11a2f61
commit 0df4747
Show file tree

Hide file tree

Showing 16 changed files with 77 additions and 51 deletions.
diff --git a/app/charts/area/chart-area.tsx b/app/charts/area/chart-area.tsx
@@ -40,6 +40,9 @@ export const ChartAreasVisualization = ({
   published: boolean;
 }) => {
   const locale = useLocale();
+  const componentIrisToFilterBy = published
+    ? getChartConfigComponentIris(chartConfig)
+    : undefined;
   const [metadataQuery] = useDataCubeMetadataQuery({
     variables: {
       iri: dataSetIri,
@@ -53,9 +56,7 @@ export const ChartAreasVisualization = ({
       iri: dataSetIri,
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
-      componentIris: published
-        ? getChartConfigComponentIris(chartConfig)
-        : undefined,
+      componentIris: componentIrisToFilterBy,
       locale,
     },
   });
@@ -65,7 +66,7 @@ export const ChartAreasVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null,
+      componentIris: componentIrisToFilterBy,
       filters: queryFilters,
     },
   });

diff --git a/app/charts/column/chart-column.tsx b/app/charts/column/chart-column.tsx
@@ -50,6 +50,9 @@ export const ChartColumnsVisualization = ({
   published: boolean;
 }) => {
   const locale = useLocale();
+  const componentIrisToFilterBy = published
+    ? getChartConfigComponentIris(chartConfig)
+    : undefined;
   const [metadataQuery] = useDataCubeMetadataQuery({
     variables: {
       iri: dataSetIri,
@@ -64,19 +67,16 @@ export const ChartColumnsVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      componentIris: published
-        ? getChartConfigComponentIris(chartConfig)
-        : undefined,
+      componentIris: componentIrisToFilterBy,
     },
   });
-
   const [observationsQuery] = useDataCubeObservationsQuery({
     variables: {
       iri: dataSetIri,
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null,
+      componentIris: componentIrisToFilterBy,
       filters: queryFilters,
     },
   });

diff --git a/app/charts/line/chart-lines.tsx b/app/charts/line/chart-lines.tsx
@@ -41,6 +41,9 @@ export const ChartLinesVisualization = ({
   published: boolean;
 }) => {
   const locale = useLocale();
+  const componentIrisToFilterBy = published
+    ? getChartConfigComponentIris(chartConfig)
+    : undefined;
   const [metadataQuery] = useDataCubeMetadataQuery({
     variables: {
       iri: dataSetIri,
@@ -55,9 +58,7 @@ export const ChartLinesVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      componentIris: published
-        ? getChartConfigComponentIris(chartConfig)
-        : undefined,
+      componentIris: componentIrisToFilterBy,
     },
   });
   const [observationsQuery] = useDataCubeObservationsQuery({
@@ -66,7 +67,7 @@ export const ChartLinesVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null,
+      componentIris: componentIrisToFilterBy,
       filters: queryFilters,
     },
   });

diff --git a/app/charts/map/chart-map.tsx b/app/charts/map/chart-map.tsx
@@ -53,6 +53,9 @@ export const ChartMapVisualization = ({
   const locale = useLocale();
   const areaDimensionIri = chartConfig.fields.areaLayer?.componentIri || "";
   const symbolDimensionIri = chartConfig.fields.symbolLayer?.componentIri || "";
+  const componentIrisToFilterBy = published
+    ? getChartConfigComponentIris(chartConfig)
+    : undefined;
   const [metadataQuery] = useDataCubeMetadataQuery({
     variables: {
       iri: dataSetIri,
@@ -67,9 +70,7 @@ export const ChartMapVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      componentIris: published
-        ? getChartConfigComponentIris(chartConfig)
-        : undefined,
+      componentIris: componentIrisToFilterBy,
     },
   });
   const [observationsQuery] = useDataCubeObservationsQuery({
@@ -78,7 +79,7 @@ export const ChartMapVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null, // FIXME: Try to load less dimensions
+      componentIris: componentIrisToFilterBy,
       filters: queryFilters,
     },
   });

diff --git a/app/charts/pie/chart-pie.tsx b/app/charts/pie/chart-pie.tsx
@@ -37,6 +37,9 @@ export const ChartPieVisualization = ({
   published: boolean;
 }) => {
   const locale = useLocale();
+  const componentIrisToFilterBy = published
+    ? getChartConfigComponentIris(chartConfig)
+    : undefined;
   const [metadataQuery] = useDataCubeMetadataQuery({
     variables: {
       iri: dataSetIri,
@@ -51,9 +54,7 @@ export const ChartPieVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      componentIris: published
-        ? getChartConfigComponentIris(chartConfig)
-        : undefined,
+      componentIris: componentIrisToFilterBy,
     },
   });
   const [observationsQuery] = useDataCubeObservationsQuery({
@@ -62,7 +63,7 @@ export const ChartPieVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null,
+      componentIris: componentIrisToFilterBy,
       filters: queryFilters,
     },
   });

diff --git a/app/charts/scatterplot/chart-scatterplot.tsx b/app/charts/scatterplot/chart-scatterplot.tsx
@@ -45,6 +45,9 @@ export const ChartScatterplotVisualization = ({
   published: boolean;
 }) => {
   const locale = useLocale();
+  const componentIrisToFilterBy = published
+    ? getChartConfigComponentIris(chartConfig)
+    : undefined;
   const [metadataQuery] = useDataCubeMetadataQuery({
     variables: {
       iri: dataSetIri,
@@ -59,9 +62,7 @@ export const ChartScatterplotVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      componentIris: published
-        ? getChartConfigComponentIris(chartConfig)
-        : undefined,
+      componentIris: componentIrisToFilterBy,
     },
   });
   const [observationsQuery] = useDataCubeObservationsQuery({
@@ -70,7 +71,7 @@ export const ChartScatterplotVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null,
+      componentIris: componentIrisToFilterBy,
       filters: queryFilters,
     },
   });

diff --git a/app/charts/table/chart-table.tsx b/app/charts/table/chart-table.tsx
@@ -26,6 +26,9 @@ export const ChartTableVisualization = ({
   published: boolean;
 }) => {
   const locale = useLocale();
+  const componentIrisToFilterBy = published
+    ? getChartConfigComponentIris(chartConfig)
+    : undefined;
   const [metadataQuery] = useDataCubeMetadataQuery({
     variables: {
       iri: dataSetIri,
@@ -40,9 +43,7 @@ export const ChartTableVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      componentIris: published
-        ? getChartConfigComponentIris(chartConfig)
-        : undefined,
+      componentIris: componentIrisToFilterBy,
     },
   });
   const [observationsQuery] = useDataCubeObservationsQuery({
@@ -51,7 +52,7 @@ export const ChartTableVisualization = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null,
+      componentIris: componentIrisToFilterBy,
       filters: chartConfig.filters,
     },
   });

diff --git a/app/components/chart-footnotes.tsx b/app/components/chart-footnotes.tsx
@@ -3,6 +3,10 @@ import { Box, Button, Link, Theme, Typography } from "@mui/material";
 import { makeStyles } from "@mui/styles";
 import { PropsWithChildren, useEffect, useMemo, useState } from "react";
 
+import {
+  getChartConfigComponentIris,
+  useQueryFilters,
+} from "@/charts/shared/chart-helpers";
 import { useChartTablePreview } from "@/components/chart-table-preview";
 import { DataDownloadMenu, RunSparqlQuery } from "@/components/data-download";
 import { ChartConfig, DataSource } from "@/configurator";
@@ -16,8 +20,6 @@ import { useLocale } from "@/locales/use-locale";
 import { useEmbedOptions } from "@/utils/embed";
 import { makeOpenDataLink } from "@/utils/opendata";
 
-import { useQueryFilters } from "../charts/shared/chart-helpers";
-
 export const useFootnotesStyles = makeStyles<Theme, { useMarginTop: boolean }>(
   (theme) => ({
     actions: {
@@ -92,13 +94,14 @@ export const ChartFootnotes = ({
 
   // Data for data download
   const filters = useQueryFilters({ chartConfig });
+  const componentIrisToFilterBy = getChartConfigComponentIris(chartConfig);
   const [{ data: visibleData }] = useDataCubeObservationsQuery({
     variables: {
       iri: dataSetIri,
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null,
+      componentIris: componentIrisToFilterBy,
       filters,
     },
   });
@@ -189,6 +192,7 @@ export const ChartFootnotes = ({
               dataSetIri={dataSetIri}
               dataSource={dataSource}
               title={dataCubeByIri.title}
+              componentIris={componentIrisToFilterBy}
               filters={filters}
             />
           ) : null}

diff --git a/app/components/data-download.tsx b/app/components/data-download.tsx
@@ -132,11 +132,13 @@ export const DataDownloadMenu = memo(
     dataSetIri,
     dataSource,
     filters,
+    componentIris,
     title,
   }: {
     dataSetIri: string;
     dataSource: DataSource;
     filters?: QueryFilters;
+    componentIris?: string[];
     title: string;
   }) => {
     return (
@@ -145,6 +147,7 @@ export const DataDownloadMenu = memo(
           dataSetIri={dataSetIri}
           dataSource={dataSource}
           fileName={title}
+          componentIris={componentIris}
           filters={filters}
         />
       </DataDownloadStateProvider>
@@ -156,11 +159,13 @@ const DataDownloadInnerMenu = ({
   dataSetIri,
   dataSource,
   fileName,
+  componentIris,
   filters,
 }: {
   dataSetIri: string;
   dataSource: DataSource;
   fileName: string;
+  componentIris?: string[];
   filters?: QueryFilters;
 }) => {
   const [state] = useDataDownloadState();
@@ -207,6 +212,7 @@ const DataDownloadInnerMenu = ({
               <Trans id="button.download.data.visible">Chart dataset</Trans>
             }
             fileName={fileName}
+            componentIris={componentIris}
             filters={filters}
           />
         )}
@@ -215,6 +221,7 @@ const DataDownloadInnerMenu = ({
           dataSource={dataSource}
           subheader={<Trans id="button.download.data.all">Full dataset</Trans>}
           fileName={fileName}
+          componentIris={componentIris}
         />
         {state.error && (
           <RawMenuItem>
@@ -233,12 +240,14 @@ const DataDownloadMenuSection = ({
   dataSource,
   subheader,
   fileName,
+  componentIris,
   filters,
 }: {
   dataSetIri: string;
   dataSource: DataSource;
   subheader: ReactNode;
   fileName: string;
+  componentIris?: string[];
   filters?: QueryFilters;
 }) => {
   return (
@@ -255,6 +264,7 @@ const DataDownloadMenuSection = ({
               dataSource={dataSource}
               fileName={fileName}
               fileFormat={fileFormat}
+              componentIris={componentIris}
               filters={filters}
             />
           ))}
@@ -269,12 +279,14 @@ const DownloadMenuItem = ({
   dataSource,
   fileName,
   fileFormat,
+  componentIris,
   filters,
 }: {
   dataSetIri: string;
   dataSource: DataSource;
   fileName: string;
   fileFormat: FileFormat;
+  componentIris?: string[];
   filters?: QueryFilters;
 }) => {
   const locale = useLocale();
@@ -333,6 +345,7 @@ const DownloadMenuItem = ({
                   sourceType: dataSource.type,
                   sourceUrl: dataSource.url,
                   locale,
+                  componentIris,
                 }
               )
               .toPromise();
@@ -346,7 +359,7 @@ const DownloadMenuItem = ({
                 sourceType: dataSource.type,
                 sourceUrl: dataSource.url,
                 locale,
-                dimensions: null,
+                componentIris,
                 filters,
               })
               .toPromise();

diff --git a/app/configurator/components/datatable.tsx b/app/configurator/components/datatable.tsx
@@ -14,7 +14,10 @@ import {
 import { ascending, descending } from "d3";
 import { useMemo, useRef, useState } from "react";
 
-import { useQueryFilters } from "@/charts/shared/chart-helpers";
+import {
+  getChartConfigComponentIris,
+  useQueryFilters,
+} from "@/charts/shared/chart-helpers";
 import { Loading } from "@/components/hint";
 import { OpenMetadataPanelWrapper } from "@/components/metadata-panel";
 import { ChartConfig, DataSource } from "@/configurator/config-types";
@@ -242,6 +245,7 @@ export const DataSetTable = ({
   sx?: SxProps<Theme>;
 }) => {
   const locale = useLocale();
+  const componentIrisToFilterBy = getChartConfigComponentIris(chartConfig);
   const filters = useQueryFilters({ chartConfig });
   const [{ data: metadataData }] = useDataCubeMetadataQuery({
     variables: {
@@ -257,6 +261,7 @@ export const DataSetTable = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
+      componentIris: componentIrisToFilterBy,
     },
   });
   const [{ data: observationsData }] = useDataCubeObservationsQuery({
@@ -265,7 +270,7 @@ export const DataSetTable = ({
       sourceType: dataSource.type,
       sourceUrl: dataSource.url,
       locale,
-      dimensions: null,
+      componentIris: componentIrisToFilterBy,
       filters,
     },
   });