Merge branch 'main' of github.com:visualize-admin/visualization-tool …

…into feat/combo-chart-improvements
visualize-admin · Oct 24, 2023 · 561e7e8 · 561e7e8
2 parents 7c8c733 + fff1c84
commit 561e7e8
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 98 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,9 @@ You can also check the [release page](https://github.com/visualize-admin/visuali
   - It's now possible to adjust Combo charts colors 🧑‍🎨
 - Fixes
   - Color picker in now in sync with selected color palette
+- Performance
+  - Improved the performance of data download
+  - (min|max)Inclusive values stored in `sh:or` are now also retrieved
 - Style
   - Color picker's design has been adjusted to be more user-friendly
   - Map now outlines shapes on hover, instead of changing their colors

diff --git a/app/.env.development b/app/.env.development
@@ -3,3 +3,4 @@ ENDPOINT=sparql+https://lindas.admin.ch/query
 SPARQL_GEO_ENDPOINT=https://geo.ld.admin.ch/query
 GRAPHQL_ENDPOINT=/api/graphql
 WHITELISTED_DATA_SOURCES=["Prod", "Int", "Test"]
+SENTRY_IGNORE_API_RESOLUTION_ERROR=1
diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx
@@ -959,7 +959,7 @@ export const DatasetResult = ({
               </Link>
             ))
           : null}
-        {creator ? (
+        {creator && creator.label ? (
           <Link
             key={creator.iri}
             href={`/browse/organization/${encodeURIComponent(creator.iri)}`}

diff --git a/app/components/data-download.tsx b/app/components/data-download.tsx
@@ -7,6 +7,7 @@ import {
   Typography,
 } from "@mui/material";
 import { ascending } from "d3";
+import { Workbook } from "exceljs";
 import { saveAs } from "file-saver";
 import keyBy from "lodash/keyBy";
 import HoverMenu from "material-ui-popup-state/HoverMenu";
@@ -25,7 +26,7 @@ import {
   useContext,
   useState,
 } from "react";
-import { OperationResult, useClient } from "urql";
+import { useClient } from "urql";
 
 import { getSortedColumns } from "@/browse/datatable";
 import Flex from "@/components/flex";
@@ -325,7 +326,7 @@ const DownloadMenuItem = ({
   const urqlClient = useClient();
   const [state, dispatch] = useDataDownloadState();
   const download = useCallback(
-    (
+    async (
       componentsData: ComponentsQuery,
       observationsData: DataCubeObservationsQuery
     ) => {
@@ -343,13 +344,29 @@ const DownloadMenuItem = ({
         dimensionParsers,
       });
 
-      return fetch("/api/download", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ columnKeys, data, fileFormat }),
-      }).then((res) =>
-        res.blob().then((blob) => saveAs(blob, `${fileName}.${fileFormat}`))
-      );
+      const workbook = new Workbook();
+      const worksheet = workbook.addWorksheet("data");
+      worksheet.columns = columnKeys.map((d) => ({
+        header: d,
+        key: d,
+      }));
+      worksheet.addRows(data);
+
+      switch (fileFormat) {
+        case "csv":
+          const csv = await workbook.csv.writeBuffer();
+          saveAs(new Blob([csv], { type: "text/csv" }), `${fileName}.csv`);
+          break;
+        case "xlsx":
+          const xlsx = await workbook.xlsx.writeBuffer();
+          saveAs(
+            new Blob([xlsx], {
+              type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            }),
+            `${fileName}.xlsx`
+          );
+          break;
+      }
     },
     [fileFormat, fileName, locale]
   );
@@ -363,8 +380,8 @@ const DownloadMenuItem = ({
         dispatch({ isDownloading: true });
 
         try {
-          const componentsResult: OperationResult<ComponentsQuery> =
-            await urqlClient
+          const [componentsResult, observationsResult] = await Promise.all([
+            urqlClient
               .query<ComponentsQuery, ComponentsQueryVariables>(
                 ComponentsDocument,
                 {
@@ -375,9 +392,8 @@ const DownloadMenuItem = ({
                   componentIris: undefined,
                 }
               )
-              .toPromise();
-          const observationsResult: OperationResult<DataCubeObservationsQuery> =
-            await urqlClient
+              .toPromise(),
+            urqlClient
               .query<
                 DataCubeObservationsQuery,
                 DataCubeObservationsQueryVariables
@@ -389,7 +405,8 @@ const DownloadMenuItem = ({
                 componentIris: undefined,
                 filters,
               })
-              .toPromise();
+              .toPromise(),
+          ]);
 
           if (componentsResult.data && observationsResult.data) {
             await download(componentsResult.data, observationsResult.data);

diff --git a/app/domain/data.ts b/app/domain/data.ts
@@ -78,7 +78,6 @@ export const parseRDFLiteral = <T = ObservationValue>(value: Literal): T => {
     case "string":
     case "boolean":
       return v as T;
-    // return v === "true" ? true : false;
     case "float":
     case "integer":
     case "long":
@@ -96,13 +95,6 @@ export const parseRDFLiteral = <T = ObservationValue>(value: Literal): T => {
     case "unsignedShort":
     case "unsignedByte":
       return +v as T;
-    // TODO: Figure out how to preserve granularity of date (maybe include interval?)
-    // case "date":
-    // case "time":
-    // case "dateTime":
-    // case "gYear":
-    // case "gYearMonth":
-    //   return new Date(v);
     default:
       return v as T;
   }

diff --git a/app/pages/api/download.ts b/app/pages/api/download.ts
diff --git a/app/rdf/queries.ts b/app/rdf/queries.ts
@@ -220,19 +220,48 @@ export const getCubeDimensionValues = async ({
 }): Promise<DimensionValue[]> => {
   const { dimension, cube, locale, data } = rdimension;
 
-  if (
-    typeof dimension.minInclusive !== "undefined" &&
-    typeof dimension.maxInclusive !== "undefined" &&
-    data.dataKind !== "Time" &&
-    data.scaleType !== "Ordinal"
-  ) {
-    const min = parseObservationValue({ value: dimension.minInclusive }) ?? 0;
-    const max = parseObservationValue({ value: dimension.maxInclusive }) ?? 0;
-
-    return [
-      { value: min, label: `${min}` },
-      { value: max, label: `${max}` },
-    ];
+  if (data.dataKind !== "Time" && data.scaleType !== "Ordinal") {
+    if (
+      typeof dimension.minInclusive !== "undefined" &&
+      typeof dimension.maxInclusive !== "undefined"
+    ) {
+      const min = parseObservationValue({ value: dimension.minInclusive }) ?? 0;
+      const max = parseObservationValue({ value: dimension.maxInclusive }) ?? 0;
+
+      return [
+        { value: min, label: `${min}` },
+        { value: max, label: `${max}` },
+      ];
+    }
+
+    // Try to get min/max values from a list of values.
+    let listItemPointer = dimension.out(ns.sh.or);
+
+    while (
+      listItemPointer.out(ns.rdf.rest).value &&
+      // Only try until we reach the end of the list.
+      !listItemPointer.out(ns.rdf.rest).term?.equals(ns.rdf.nil)
+    ) {
+      const item = listItemPointer.out(ns.rdf.first);
+      const itemMin = item.out(ns.sh.minInclusive);
+      const itemMax = item.out(ns.sh.maxInclusive);
+
+      if (
+        typeof itemMin.value !== "undefined" &&
+        typeof itemMax.value !== "undefined"
+      ) {
+        const min = +itemMin.value;
+        const max = +itemMax.value;
+
+        return [
+          { value: min, label: `${min}` },
+          { value: max, label: `${max}` },
+        ];
+      }
+
+      // Move to next list item.
+      listItemPointer = listItemPointer.out(ns.rdf.rest);
+    }
   }
 
   if (shouldLoadMinMaxValues(rdimension)) {