Merge pull request #377 from visualize-admin/perf/filters

feat: Only load relevant values in right filter

app/charts/area/chart-area.tsx

@@ -45,7 +45,7 @@ export const ChartAreasVisualization = ({
     variables: {
       locale,
       iri: dataSetIri,
-      measures: [chartConfig.fields.y.componentIri], // FIXME: Other fields may also be measures
+      dimensions: null, // FIXME: Try to load less dimensions
       filters: queryFilters,
     },
   });

app/charts/bar/chart-bar.tsx

@@ -43,7 +43,7 @@ export const ChartBarsVisualization = ({
     variables: {
       locale,
       iri: dataSetIri,
-      measures: [chartConfig.fields.x.componentIri], // FIXME: Other fields may also be measures
+      dimensions: null, // FIXME: Try to load less dimensions
       filters: queryFilters,
     },
   });

app/charts/column/chart-column.tsx

@@ -52,7 +52,7 @@ export const ChartColumnsVisualization = ({
     variables: {
       locale,
       iri: dataSetIri,
-      measures: [chartConfig.fields.y.componentIri], // FIXME: Other fields may also be measures
+      dimensions: null,
       filters: queryFilters,
     },
   });

app/charts/line/chart-lines.tsx

@@ -48,7 +48,7 @@ export const ChartLinesVisualization = ({
     variables: {
       locale,
       iri: dataSetIri,
-      measures: [chartConfig.fields.y.componentIri], // FIXME: Other fields may also be measures
+      dimensions: null, // FIXME: Try to load less dimensions
       filters: queryFilters,
     },
   });

app/charts/map/chart-map.tsx

@@ -50,10 +50,7 @@ export const ChartMapVisualization = ({
     variables: {
       locale,
       iri: dataSetIri,
-      measures: [
-        chartConfig.fields.areaLayer.measureIri,
-        chartConfig.fields.symbolLayer.measureIri,
-      ],
+      dimensions: null, // FIXME: Try to load less dimensions
       filters: queryFilters,
     },
   });

app/charts/pie/chart-pie.tsx

@@ -42,7 +42,7 @@ export const ChartPieVisualization = ({
     variables: {
       locale,
       iri: dataSetIri,
-      measures: [chartConfig.fields.y.componentIri], // FIXME: Other fields may also be measures
+      dimensions: null,
       filters: queryFilters,
     },
   });

app/charts/scatterplot/chart-scatterplot.tsx

@@ -50,10 +50,7 @@ export const ChartScatterplotVisualization = ({
     variables: {
       locale,
       iri: dataSetIri,
-      measures: [
-        chartConfig.fields.x.componentIri,
-        chartConfig.fields.y.componentIri,
-      ], // FIXME: Other fields may also be measures
+      dimensions: null, // FIXME: Other fields may also be measures
       filters: queryFilters,
     },
   });

app/charts/table/chart-table.tsx

@@ -26,17 +26,11 @@ export const ChartTableVisualization = ({
 }) => {
   const locale = useLocale();
 
-  const measures = Object.keys(chartConfig.fields).filter(
-    (key) =>
-      chartConfig.fields[key].componentType === "Measure" &&
-      !chartConfig.fields[key].isHidden
-  );
-
   const [{ data, fetching, error }] = useDataCubeObservationsQuery({
     variables: {
       locale,
       iri: dataSetIri,
-      measures,
+      dimensions: null,
       filters: chartConfig.filters,
     },
   });

app/components/data-download.tsx

@@ -4,12 +4,7 @@ import { saveAs } from "file-saver";
 import { memo, ReactNode, useMemo } from "react";
 import { Box, Button, Link } from "theme-ui";
 import { useQueryFilters } from "../charts/shared/chart-helpers";
-import {
-  ChartConfig,
-  ChartFields,
-  isMapConfig,
-  isTableConfig,
-} from "../configurator";
+import { ChartConfig, ChartFields } from "../configurator";
 import { Observation } from "../domain/data";
 import {
   DimensionMetaDataFragment,
@@ -30,24 +25,7 @@ export const DataDownload = memo(
     chartConfig: ChartConfig;
   }) => {
     const locale = useLocale();
-    const measures = useMemo(
-      () =>
-        "y" in chartConfig.fields
-          ? [chartConfig.fields.y.componentIri]
-          : isTableConfig(chartConfig)
-          ? Object.values(chartConfig.fields).flatMap((f) =>
-              f.componentType === "Measure" && !f.isHidden
-                ? [f.componentIri]
-                : []
-            )
-          : isMapConfig(chartConfig)
-          ? [
-              chartConfig.fields.areaLayer.measureIri,
-              chartConfig.fields.symbolLayer.measureIri,
-            ]
-          : [],
-      [chartConfig]
-    );
+
     const filters = useQueryFilters({
       chartConfig,
     });
@@ -56,7 +34,7 @@ export const DataDownload = memo(
       variables: {
         locale,
         iri: dataSetIri,
-        measures, // FIXME: Other fields may also be measures
+        dimensions: null, // FIXME: Other fields may also be measures
         filters,
       },
     });

app/configurator/components/chart-options-selector.tsx

@@ -75,7 +75,7 @@ export const ChartOptionsSelector = ({
     variables: {
       locale,
       iri: state.dataSet,
-      measures,
+      dimensions: null,
       filters: state.chartConfig.filters,
     },
   });

app/configurator/components/datatable.tsx

@@ -123,7 +123,7 @@ export const DataSetPreviewTable = ({
     variables: {
       iri: dataSetIri,
       locale,
-      measures: measures.map((m) => m.iri),
+      dimensions: null,
     },
   });
 

app/domain/data.ts

@@ -7,6 +7,7 @@ import {
   NominalDimension,
   OrdinalDimension,
 } from "../graphql/query-hooks";
+import { ResolvedDimension } from "../graphql/shared-types";
 
 export type RawObservationValue = Literal | NamedNode;
 
@@ -202,3 +203,15 @@ export const isGeoShapesDimension = (
 ): dimension is GeoShapesDimension => {
   return dimension?.__typename === "GeoShapesDimension";
 };
+
+export const isStandardErrorResolvedDimension = (dim: ResolvedDimension) => {
+  return dim.data?.related.some((x) => x.type === "StandardError");
+};
+
+export const shouldValuesBeLoadedForResolvedDimension = (
+  dim: ResolvedDimension
+) => {
+  return !(
+    dim.data.isMeasureDimension || isStandardErrorResolvedDimension(dim)
+  );
+};

app/graphql/queries/data-cubes.graphql

@@ -71,11 +71,11 @@ query DataCubePreview(
 query DataCubePreviewObservations(
   $iri: String!
   $locale: String!
-  $measures: [String!]!
+  $dimensions: [String!]
   $latest: Boolean
 ) {
   dataCubeByIri(iri: $iri, locale: $locale, latest: $latest) {
-    observations(limit: 10, measures: $measures) {
+    observations(limit: 10, dimensions: $dimensions) {
       data
       sparql
     }
@@ -195,7 +195,7 @@ query TemporalDimensionValues(
 query DataCubeObservations(
   $iri: String!
   $locale: String!
-  $measures: [String!]!
+  $dimensions: [String!]
   $filters: Filters
   $latest: Boolean
 ) {
@@ -209,7 +209,7 @@ query DataCubeObservations(
     measures {
       ...dimensionMetaData
     }
-    observations(measures: $measures, filters: $filters) {
+    observations(dimensions: $dimensions, filters: $filters) {
       data
       sparqlEditorUrl
     }

app/graphql/query-hooks.ts

@@ -44,7 +44,7 @@ export type DataCube = {
 
 export type DataCubeObservationsArgs = {
   limit?: Maybe<Scalars['Int']>;
-  measures?: Maybe<Array<Scalars['String']>>;
+  dimensions?: Maybe<Array<Scalars['String']>>;
   filters?: Maybe<Scalars['Filters']>;
 };
 
@@ -375,7 +375,7 @@ export type DataCubePreviewQuery = { __typename: 'Query', dataCubeByIri?: Maybe<
 export type DataCubePreviewObservationsQueryVariables = Exact<{
   iri: Scalars['String'];
   locale: Scalars['String'];
-  measures: Array<Scalars['String']> | Scalars['String'];
+  dimensions?: Maybe<Array<Scalars['String']> | Scalars['String']>;
   latest?: Maybe<Scalars['Boolean']>;
 }>;
 
@@ -488,7 +488,7 @@ export type TemporalDimensionValuesQuery = { __typename: 'Query', dataCubeByIri?
 export type DataCubeObservationsQueryVariables = Exact<{
   iri: Scalars['String'];
   locale: Scalars['String'];
-  measures: Array<Scalars['String']> | Scalars['String'];
+  dimensions?: Maybe<Array<Scalars['String']> | Scalars['String']>;
   filters?: Maybe<Scalars['Filters']>;
   latest?: Maybe<Scalars['Boolean']>;
 }>;
@@ -627,9 +627,9 @@ export function useDataCubePreviewQuery(options: Omit<Urql.UseQueryArgs<DataCube
   return Urql.useQuery<DataCubePreviewQuery>({ query: DataCubePreviewDocument, ...options });
 };
 export const DataCubePreviewObservationsDocument = gql`
-    query DataCubePreviewObservations($iri: String!, $locale: String!, $measures: [String!]!, $latest: Boolean) {
+    query DataCubePreviewObservations($iri: String!, $locale: String!, $dimensions: [String!], $latest: Boolean) {
   dataCubeByIri(iri: $iri, locale: $locale, latest: $latest) {
-    observations(limit: 10, measures: $measures) {
+    observations(limit: 10, dimensions: $dimensions) {
       data
       sparql
     }
@@ -754,7 +754,7 @@ export function useTemporalDimensionValuesQuery(options: Omit<Urql.UseQueryArgs<
   return Urql.useQuery<TemporalDimensionValuesQuery>({ query: TemporalDimensionValuesDocument, ...options });
 };
 export const DataCubeObservationsDocument = gql`
-    query DataCubeObservations($iri: String!, $locale: String!, $measures: [String!]!, $filters: Filters, $latest: Boolean) {
+    query DataCubeObservations($iri: String!, $locale: String!, $dimensions: [String!], $filters: Filters, $latest: Boolean) {
   dataCubeByIri(iri: $iri, locale: $locale, latest: $latest) {
     iri
     title
@@ -765,7 +765,7 @@ export const DataCubeObservationsDocument = gql`
     measures {
       ...dimensionMetaData
     }
-    observations(measures: $measures, filters: $filters) {
+    observations(dimensions: $dimensions, filters: $filters) {
       data
       sparqlEditorUrl
     }

app/graphql/resolver-types.ts

@@ -49,7 +49,7 @@ export type DataCube = {
 
 export type DataCubeObservationsArgs = {
   limit?: Maybe<Scalars['Int']>;
-  measures?: Maybe<Array<Scalars['String']>>;
+  dimensions?: Maybe<Array<Scalars['String']>>;
   filters?: Maybe<Scalars['Filters']>;
 };
 

app/graphql/resolvers.ts

@@ -62,6 +62,7 @@ const Query: QueryResolvers = {
         filters: queryFilters,
         limit: 1,
         raw: true,
+        dimensions: null,
       });
       if (obs.length === 0) {
         continue;
@@ -240,12 +241,13 @@ const DataCube: DataCubeResolvers = {
 
     return dimension ?? null;
   },
-  observations: async ({ cube, locale }, { limit, filters, measures }) => {
+  observations: async ({ cube, locale }, { limit, filters, dimensions }) => {
     const { query, observations, observationsRaw } = await getCubeObservations({
       cube,
       locale,
       filters: filters ?? undefined,
       limit: limit ?? undefined,
+      dimensions,
     });
 
     // const constructedFilters = filters

app/graphql/schema.graphql

@@ -36,7 +36,7 @@ type DataCube {
   publicationStatus: DataCubePublicationStatus!
   observations(
     limit: Int
-    measures: [String!]
+    dimensions: [String!]
     filters: Filters
   ): ObservationsQuery!
   dimensions: [Dimension!]!

app/rdf/queries.ts

@@ -1,4 +1,5 @@
 import { descending, group, index } from "d3";
+import { Maybe } from "graphql-tools";
 import {
   Cube,
   CubeDimension,
@@ -14,6 +15,7 @@ import {
   DimensionValue,
   Observation,
   parseObservationValue,
+  shouldValuesBeLoadedForResolvedDimension,
 } from "../domain/data";
 import { SPARQL_EDITOR, SPARQL_ENDPOINT } from "../domain/env";
 import { DataCubeSearchFilter, DataCubeTheme } from "../graphql/query-hooks";
@@ -250,9 +252,10 @@ export const createCubeDimensionValuesLoader =
   };
 
 export const getCubeDimensionValues = async (
-  { dimension, cube, locale, data }: ResolvedDimension,
+  rdimension: ResolvedDimension,
   filters?: Filters
 ): Promise<DimensionValue[]> => {
+  const { dimension, cube, locale, data } = rdimension;
   if (
     typeof dimension.minInclusive !== "undefined" &&
     typeof dimension.maxInclusive !== "undefined" &&
@@ -267,6 +270,10 @@ export const getCubeDimensionValues = async (
     ];
   }
 
+  if (!shouldValuesBeLoadedForResolvedDimension(rdimension)) {
+    return [];
+  }
+
   return await getCubeDimensionValuesWithLabels({
     dimension,
     cube,
@@ -394,6 +401,7 @@ export const getCubeObservations = async ({
   filters,
   limit,
   raw,
+  dimensions,
 }: {
   cube: Cube;
   locale: string;
@@ -403,6 +411,7 @@ export const getCubeObservations = async ({
   limit?: number;
   /** Returns IRIs instead of labels for NamedNodes  */
   raw?: boolean;
+  dimensions: Maybe<string[]> | undefined;
 }): Promise<{
   query: string;
   observations: Observation[];
@@ -417,7 +426,8 @@ export const getCubeObservations = async ({
         cd.path &&
         ![ns.rdf.type.value, ns.cube.observedBy.value].includes(
           cd.path.value ?? ""
-        )
+        ) &&
+        (dimensions ? dimensions.includes(cd.path.value) : true)
     )
   );
 
@@ -428,7 +438,10 @@ export const getCubeObservations = async ({
   /**
    * Add labels to named dimensions
    */
-  const cubeDimensions = await getCubeDimensions({ cube, locale });
+  const allCubeDimensions = await getCubeDimensions({ cube, locale });
+  const cubeDimensions = allCubeDimensions.filter((d) =>
+    dimensions ? dimensions.includes(d.data.iri) : true
+  );
 
   // Find dimensions which are NOT literal
   const namedDimensions = cubeDimensions.filter(

app/utils/dimension-hierarchy.ts

@@ -277,7 +277,7 @@ export const useHierarchicalDimensionValuesQuery = ({
     variables: {
       iri: dataSetIri,
       locale,
-      measures: path,
+      dimensions: path,
     },
   });