Merge pull request #1231 from visualize-admin/perf/move-data-download…

…-to-api-route

perf: Improve data download performance

CHANGELOG.md

@@ -10,7 +10,8 @@ You can also check the [release page](https://github.com/visualize-admin/visuali
 ## Unreleased
 
 - Performance
-  - (min & max)Inclusive values stored in `sh:or` are now also retrieved
+  - Improved the performance of data download
+  - (min|max)Inclusive values stored in `sh:or` are now also retrieved
 - Style
   - Map now outlines shapes on hover, instead of changing their colors
 - Maintenance

app/.env.development

@@ -3,3 +3,4 @@ ENDPOINT=sparql+https://lindas.admin.ch/query
 SPARQL_GEO_ENDPOINT=https://geo.ld.admin.ch/query
 GRAPHQL_ENDPOINT=/api/graphql
 WHITELISTED_DATA_SOURCES=["Prod", "Int", "Test"]
+SENTRY_IGNORE_API_RESOLUTION_ERROR=1

app/browser/dataset-browse.tsx

@@ -959,7 +959,7 @@ export const DatasetResult = ({
               </Link>
             ))
           : null}
-        {creator ? (
+        {creator && creator.label ? (
           <Link
             key={creator.iri}
             href={`/browse/organization/${encodeURIComponent(creator.iri)}`}

app/components/data-download.tsx

@@ -7,6 +7,7 @@ import {
   Typography,
 } from "@mui/material";
 import { ascending } from "d3";
+import { Workbook } from "exceljs";
 import { saveAs } from "file-saver";
 import keyBy from "lodash/keyBy";
 import HoverMenu from "material-ui-popup-state/HoverMenu";
@@ -25,7 +26,7 @@ import {
   useContext,
   useState,
 } from "react";
-import { OperationResult, useClient } from "urql";
+import { useClient } from "urql";
 
 import { getSortedColumns } from "@/browse/datatable";
 import Flex from "@/components/flex";
@@ -325,7 +326,7 @@ const DownloadMenuItem = ({
   const urqlClient = useClient();
   const [state, dispatch] = useDataDownloadState();
   const download = useCallback(
-    (
+    async (
       componentsData: ComponentsQuery,
       observationsData: DataCubeObservationsQuery
     ) => {
@@ -343,13 +344,29 @@ const DownloadMenuItem = ({
         dimensionParsers,
       });
 
-      return fetch("/api/download", {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ columnKeys, data, fileFormat }),
-      }).then((res) =>
-        res.blob().then((blob) => saveAs(blob, `${fileName}.${fileFormat}`))
-      );
+      const workbook = new Workbook();
+      const worksheet = workbook.addWorksheet("data");
+      worksheet.columns = columnKeys.map((d) => ({
+        header: d,
+        key: d,
+      }));
+      worksheet.addRows(data);
+
+      switch (fileFormat) {
+        case "csv":
+          const csv = await workbook.csv.writeBuffer();
+          saveAs(new Blob([csv], { type: "text/csv" }), `${fileName}.csv`);
+          break;
+        case "xlsx":
+          const xlsx = await workbook.xlsx.writeBuffer();
+          saveAs(
+            new Blob([xlsx], {
+              type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            }),
+            `${fileName}.xlsx`
+          );
+          break;
+      }
     },
     [fileFormat, fileName, locale]
   );
@@ -363,8 +380,8 @@ const DownloadMenuItem = ({
         dispatch({ isDownloading: true });
 
         try {
-          const componentsResult: OperationResult<ComponentsQuery> =
-            await urqlClient
+          const [componentsResult, observationsResult] = await Promise.all([
+            urqlClient
               .query<ComponentsQuery, ComponentsQueryVariables>(
                 ComponentsDocument,
                 {
@@ -375,9 +392,8 @@ const DownloadMenuItem = ({
                   componentIris: undefined,
                 }
               )
-              .toPromise();
-          const observationsResult: OperationResult<DataCubeObservationsQuery> =
-            await urqlClient
+              .toPromise(),
+            urqlClient
               .query<
                 DataCubeObservationsQuery,
                 DataCubeObservationsQueryVariables
@@ -389,7 +405,8 @@ const DownloadMenuItem = ({
                 componentIris: undefined,
                 filters,
               })
-              .toPromise();
+              .toPromise(),
+          ]);
 
           if (componentsResult.data && observationsResult.data) {
             await download(componentsResult.data, observationsResult.data);