Merge pull request #955 from visualize-admin/feat/lru-cache

app/gql-flamegraph/devtool.tsx

@@ -30,13 +30,24 @@ import { Exchange, Operation, OperationResult } from "urql";
 import { pipe, tap } from "wonka";
 
 import useDisclosure from "@/configurator/components/use-disclosure";
+import { RequestQueryMeta } from "@/graphql/query-meta";
 import useEvent from "@/utils/use-event";
 
 export type Timings = Record<
   string,
   { start: number; end: number; children?: Timings }
 >;
 
+export type VisualizeOperationResult<TData = any, TVariables = any> = Omit<
+  OperationResult<TData, TVariables>,
+  "extensions"
+> & {
+  extensions: {
+    queries: RequestQueryMeta[];
+    timings: Timings;
+  };
+};
+
 const visit = (
   t: Timings,
   visit: (item: Omit<Timings, "children">, parents: readonly string[]) => void
@@ -165,13 +176,15 @@ const AccordionOperation = ({
   end,
   ...accordionProps
 }: {
-  result: OperationResult | undefined;
+  result: VisualizeOperationResult | undefined;
   operation: Operation;
   start: number;
   end: number;
 } & Omit<AccordionProps, "children">) => {
   const duration = useMemo(() => {
-    const all = flatten(result?.extensions?.timings).sort(byStart);
+    const all = result?.extensions?.timings
+      ? flatten(result?.extensions?.timings).sort(byStart)
+      : [];
     if (all.length === 0) {
       return 0;
     }
@@ -248,7 +261,11 @@ const AccordionOperation = ({
               <Typography variant="h5" gutterBottom>
                 SPARQL queries ({result?.extensions?.queries.length})
               </Typography>
-              <Queries queries={result?.extensions?.queries} />
+              <Queries
+                queries={sortBy(result?.extensions?.queries, (q) => {
+                  return -(q.endTime - q.startTime);
+                })}
+              />
             </div>
           </Box>
         </AccordionDetails>
@@ -322,15 +339,15 @@ const useStyles = makeStyles((theme: Theme) => ({
 }));
 
 function GqlDebug() {
-  const [results, setResults] = useState([] as OperationResult[]);
+  const [results, setResults] = useState([] as VisualizeOperationResult[]);
   const opsStartMapRef = useRef(new Map<Operation["key"], number>());
   const opsEndMapRef = useRef(new Map<Operation["key"], number>());
   useEffect(() => {
     const handleOperation = (operation: Operation) => {
       opsStartMapRef.current.set(operation.key, Date.now());
       opsEndMapRef.current.set(operation.key, Date.now());
     };
-    const handleResult = (result: OperationResult) => {
+    const handleResult = (result: VisualizeOperationResult) => {
       opsEndMapRef.current.set(result.operation.key, Date.now());
       // Calls setState out of band since handleResult can be called while
       // rendering a component. setState cannot be called while rendering
@@ -408,7 +425,7 @@ function GqlDebug() {
  */
 export const urqlEE = mitt<{
   "urql-received-operation": Operation<any, any>;
-  "urql-received-result": OperationResult<any, any>;
+  "urql-received-result": VisualizeOperationResult<any, any>;
 }>();
 
 export const gqlFlamegraphExchange: Exchange = ({ forward }) => {
@@ -417,7 +434,9 @@ export const gqlFlamegraphExchange: Exchange = ({ forward }) => {
       ops$,
       tap((operation) => urqlEE.emit("urql-received-operation", operation)),
       forward,
-      tap((result) => urqlEE.emit("urql-received-result", result))
+      tap((result) =>
+        urqlEE.emit("urql-received-result", result as VisualizeOperationResult)
+      )
     );
 };
 

app/package.json

@@ -109,6 +109,7 @@
     "superjson": "^1.11.0",
     "topojson-client": "^3.1.0",
     "topojson-server": "^3.0.1",
+    "typescript-lru-cache": "^2.0.0",
     "urql": "^2.0.5",
     "use-debounce": "^7.0.0",
     "use-immer": "^0.5.1",

app/rdf/batch-load.ts

@@ -1,10 +1,22 @@
-import { SparqlQueryExecutable } from "@tpluscode/sparql-builder/lib";
+import {
+  SparqlQuery,
+  SparqlQueryExecutable,
+} from "@tpluscode/sparql-builder/lib";
 import { groups } from "d3";
 import { NamedNode, Term } from "rdf-js";
 import ParsingClient from "sparql-http-client/ParsingClient";
 
+import { makeExecuteWithCache } from "./query-cache";
+
 const BATCH_SIZE = 500;
 
+const executeWithCache = makeExecuteWithCache({
+  parse: (t) => t,
+  cacheOptions: {
+    maxSize: 10_000,
+  },
+});
+
 export default async function batchLoad<
   TReturn extends unknown,
   TId extends Term | NamedNode = Term
@@ -16,7 +28,10 @@ export default async function batchLoad<
 }: {
   ids: TId[];
   sparqlClient: ParsingClient;
-  buildQuery: (values: TId[], key: number) => SparqlQueryExecutable;
+  buildQuery: (
+    values: TId[],
+    key: number
+  ) => SparqlQueryExecutable & SparqlQuery;
   batchSize?: number;
 }): Promise<TReturn[]> {
   const batched = groups(ids, (_, i) => Math.floor(i / batchSize));
@@ -26,9 +41,10 @@ export default async function batchLoad<
       const query = buildQuery(values, key);
 
       try {
-        return (await query.execute(sparqlClient.query, {
-          operation: "postUrlencoded",
-        })) as unknown as TReturn[];
+        return (await executeWithCache(
+          sparqlClient,
+          query
+        )) as unknown as TReturn[];
       } catch (e) {
         console.log(
           `Error while querying. First ID of ${ids.length}: <${ids[0].value}>`

app/rdf/queries.ts

@@ -218,9 +218,7 @@ export const getCubeDimensionValues = async ({
       });
 
       if (result) {
-        const { minValue, maxValue } = result;
-        const min = parseObservationValue({ value: minValue }) ?? 0;
-        const max = parseObservationValue({ value: maxValue }) ?? 0;
+        const [min, max] = result;
 
         return [
           { value: min, label: `${min}` },

app/rdf/query-cache.ts

@@ -0,0 +1,39 @@
+import {
+  SparqlQuery,
+  SparqlQueryExecutable,
+} from "@tpluscode/sparql-builder/lib";
+import StreamClient from "sparql-http-client";
+import { ParsingClient } from "sparql-http-client/ParsingClient";
+import { LRUCache, LRUCacheOptions } from "typescript-lru-cache";
+
+type SparqlClient = StreamClient | ParsingClient;
+
+export const makeExecuteWithCache = <T>({
+  parse,
+  cacheOptions,
+}: {
+  parse: (v: any) => T;
+  cacheOptions: LRUCacheOptions<string, T>;
+}) => {
+  const cache = new LRUCache({
+    entryExpirationTimeInMS: 60 * 10_000,
+    ...cacheOptions,
+  });
+  return async (
+    sparqlClient: SparqlClient,
+    query: SparqlQuery & SparqlQueryExecutable
+  ) => {
+    const key = `${sparqlClient.query.endpoint.endpointUrl} - ${query.build()}`;
+    const cached = cache.get(key);
+    if (cached) {
+      return cached;
+    } else {
+      const result = await query.execute(sparqlClient.query, {
+        operation: "postUrlencoded",
+      });
+      const parsed = parse(result) as T;
+      cache.set(key, parsed);
+      return parsed;
+    }
+  };
+};

app/rdf/query-dimension-values.ts

@@ -1,10 +1,13 @@
 import { SELECT, sparql } from "@tpluscode/sparql-builder";
 import keyBy from "lodash/keyBy";
 import mapValues from "lodash/mapValues";
+import sortBy from "lodash/sortBy";
 import { Cube, CubeDimension } from "rdf-cube-view-query";
+import LiteralExt from "rdf-ext/lib/Literal";
 import { Literal, NamedNode, Term } from "rdf-js";
 import { ParsingClient } from "sparql-http-client/ParsingClient";
 
+import { parseObservationValue } from "@/domain/data";
 import { pragmas } from "@/rdf/create-source";
 
 import { Filters } from "../configurator";
@@ -13,6 +16,8 @@ import { cube as cubeNs } from "./namespace";
 import * as ns from "./namespace";
 import { parseDimensionDatatype } from "./parse";
 import { dimensionIsVersioned } from "./queries";
+import { makeExecuteWithCache } from "./query-cache";
+
 interface DimensionValue {
   value: Literal | NamedNode<string>;
 }
@@ -99,6 +104,18 @@ export async function unversionObservation({
   return mapValues(observation, (v) => (v ? unversionedIndex[v] || v : v));
 }
 
+const getFilterOrder = (filter: Filters[number]) => {
+  if (filter.type !== "single") {
+    return 0;
+  } else {
+    // Heuristic to put non discriminant filter at the end
+    // Seems like we could also do it based on the column order
+    return filter.value.toString().startsWith("https://ld.admin.ch")
+      ? Infinity
+      : 0;
+  }
+};
+
 /**
  * Load dimension values.
  *
@@ -125,9 +142,12 @@ export async function loadDimensionValues(
 
   // Consider filters before the current filter to fetch the values for
   // the current filter
-  const filterList = allFiltersList.slice(
-    0,
-    allFiltersList.findIndex(([iri]) => iri == dimensionIri?.value)
+  const filterList = sortBy(
+    allFiltersList.slice(
+      0,
+      allFiltersList.findIndex(([iri]) => iri == dimensionIri?.value)
+    ),
+    ([, filterValue]) => getFilterOrder(filterValue)
   );
 
   const query = SELECT.DISTINCT`?value`.WHERE`
@@ -181,11 +201,22 @@ export async function loadDimensionValues(
   }
 }
 
-type MinMaxResult = {
-  minValue: Literal;
-  maxValue: Literal;
+type MinMaxResult = [{ minValue: LiteralExt; maxValue: LiteralExt }];
+
+const parseMinMax = (result: MinMaxResult) => {
+  const { minValue, maxValue } = result[0];
+  const min = parseObservationValue({ value: minValue }) ?? 0;
+  const max = parseObservationValue({ value: maxValue }) ?? 0;
+  return [min, max] as const;
 };
 
+const executeMinMaxQueryWithCache = makeExecuteWithCache({
+  cacheOptions: {
+    maxSize: 10_000,
+  },
+  parse: parseMinMax,
+});
+
 export const loadMinMaxDimensionValues = async ({
   datasetIri,
   dimensionIri,
@@ -194,27 +225,24 @@ export const loadMinMaxDimensionValues = async ({
   datasetIri: Term;
   dimensionIri: Term;
   sparqlClient: ParsingClient;
-}): Promise<MinMaxResult | undefined> => {
+}) => {
   const query = SELECT`(MIN(?value) as ?minValue) (MAX(?value) as ?maxValue)`
     .WHERE`
     ${datasetIri} ${cubeNs.observationSet} ?observationSet .
     ?observationSet ${cubeNs.observation} ?observation .
     ?observation ${dimensionIri} ?value .
 
-    FILTER (STRLEN(STR(?value)) > 0)
+    FILTER (
+      (STRLEN(STR(?value)) > 0) && (STR(?value) != "NaN")
+    )
   `;
 
-  let result: MinMaxResult[] = [];
-
   try {
-    result = (await query.execute(sparqlClient.query, {
-      operation: "postUrlencoded",
-    })) as unknown as MinMaxResult[];
+    const result = await executeMinMaxQueryWithCache(sparqlClient, query);
+    return result;
   } catch {
     console.warn(
       `Failed to fetch min max dimension values for ${datasetIri}, ${dimensionIri}.`
     );
-  } finally {
-    return result[0];
   }
 };

app/rdf/query-hierarchies.ts

@@ -72,15 +72,19 @@ const toTree = (
   return sortChildren(results.map((r) => serializeNode(r, 0)).filter(truthy));
 };
 
-const findHierarchiesForDimension = (cube: Cube, dimensionIri: string) => {
+const findHierarchiesForDimension = (
+  cube: Cube,
+  dimensionIri: string,
+  locale: string
+) => {
   const newHierarchies = uniqBy(
     cube.ptr
       .any()
       .has(ns.sh.path, rdf.namedNode(dimensionIri))
       .has(ns.cubeMeta.inHierarchy)
       .out(ns.cubeMeta.inHierarchy)
       .toArray(),
-    (x) => x.value
+    (x) => getName(x, locale)
   );
   if (newHierarchies) {
     return newHierarchies;
@@ -105,7 +109,8 @@ export const queryHierarchy = async (
 ): Promise<HierarchyValue[] | null> => {
   const hierarchies = findHierarchiesForDimension(
     rdimension.cube,
-    rdimension.data.iri
+    rdimension.data.iri,
+    locale
   );
 
   if (hierarchies.length === 0) {
@@ -133,9 +138,11 @@ export const queryHierarchy = async (
       locale
     );
 
-    // Augment hierarchy value with hierarchyName so that when regrouping
-    // below, we can create the fake nodes
-    tree[0].hierarchyName = h.hierarchyName;
+    if (tree.length > 0) {
+      // Augment hierarchy value with hierarchyName so that when regrouping
+      // below, we can create the fake nodes
+      tree[0].hierarchyName = h.hierarchyName;
+    }
     return tree;
   });
 

app/utils/chart-config/versioning.ts

@@ -1,7 +1,5 @@
 import produce from "immer";
 
-import { ChartConfig } from "@/configurator";
-
 export const CHART_CONFIG_VERSION = "1.3.0";
 
 type Migration = {
@@ -486,7 +484,7 @@ export const migrateChartConfig = (
     fromVersion,
     toVersion = CHART_CONFIG_VERSION,
   }: { fromVersion?: string; toVersion?: string } = {}
-): ChartConfig => {
+) => {
   const _migrateChartConfig = (
     config: any,
     {