Merge branch 'main' of github.com:visualize-admin/visualization-tool …

…into feat/common-joinby-dimension

CHANGELOG.md

@@ -9,9 +9,17 @@ You can also check the [release page](https://github.com/visualize-admin/visuali
 
 ## Unreleased
 
+- Features
+  - Localized cube landing pages are now supported (dcat:landingPage) 🌎
+
+# [3.24.2] - 2023-11-28
+
+- Features
+  - Implemented initial version of merging the cubes (not yet exposed through UI)
 - Fixes
   - Conslidated behavior of setting initial filters (top-most hierarchy value) when filter was not present and multi-filter was removed
   - Fixed switching between segmentation dimensions in column charts
+  - Added UTF-8 formatting to CSV and XLSX files (data download)
 
 # [3.24.1] - 2023-11-13
 

app/charts/column/columns-stacked-state.tsx

@@ -429,7 +429,7 @@ const useColumnsStackedState = (
       });
 
       const xAnchorRaw = (xScale(x) as number) + bw * 0.5;
-      const yAnchor = yScale(sum(yValues.map((d) => Math.abs(d ?? 0))) * 0.5);
+      const yAnchor = yScale(sum(yValues.map((d) => d ?? 0)) * 0.5);
       const placement = getCenteredTooltipPlacement({
         chartWidth,
         xAnchor: xAnchorRaw,

app/charts/map/helpers.spec.ts

@@ -1,4 +1,4 @@
-import { renderHook } from "@testing-library/react-hooks";
+import { renderHook } from "@testing-library/react";
 
 import { BBox } from "@/config-types";
 
@@ -22,8 +22,8 @@ const featuresBBox = [
 describe("useViewState", () => {
   it("should properly set defaultViewState", () => {
     const { result, rerender } = renderHook<
-      ViewStateInitializationProps,
-      ReturnType<typeof useViewState>
+      ReturnType<typeof useViewState>,
+      ViewStateInitializationProps
     >((props: ViewStateInitializationProps) => useViewState(props), {
       initialProps: {
         width,
@@ -49,8 +49,8 @@ describe("useViewState", () => {
 
   it("should properly set viewState", () => {
     const { result } = renderHook<
-      ViewStateInitializationProps,
-      ReturnType<typeof useViewState>
+      ReturnType<typeof useViewState>,
+      ViewStateInitializationProps
     >((props: ViewStateInitializationProps) => useViewState(props), {
       initialProps: {
         width,
@@ -66,8 +66,8 @@ describe("useViewState", () => {
     );
 
     const { result: resultLocked } = renderHook<
-      ViewStateInitializationProps,
-      ReturnType<typeof useViewState>
+      ReturnType<typeof useViewState>,
+      ViewStateInitializationProps
     >((props: ViewStateInitializationProps) => useViewState(props), {
       initialProps: {
         width,

app/charts/shared/use-observation-labels.spec.ts

@@ -1,4 +1,4 @@
-import { renderHook } from "@testing-library/react-hooks";
+import { renderHook } from "@testing-library/react";
 
 import { useObservationLabels } from "@/charts/shared/observation-labels";
 import { Observation } from "@/domain/data";

app/components/data-download.tsx

@@ -238,14 +238,16 @@ const DataDownloadInnerMenu = ({
           sx: { width: 200, pt: 1, pb: 2 },
         }}
       >
-        <DataDownloadMenuSection
-          dataSource={dataSource}
-          subheader={
-            <Trans id="button.download.data.visible">Chart dataset</Trans>
-          }
-          fileName={`${fileName}-filtered`}
-          filters={filters}
-        />
+        {filters?.some((f) => f.filters) && (
+          <DataDownloadMenuSection
+            dataSource={dataSource}
+            subheader={
+              <Trans id="button.download.data.visible">Chart dataset</Trans>
+            }
+            fileName={`${fileName}-filtered`}
+            filters={filters}
+          />
+        )}
         <DataDownloadMenuSection
           dataSource={dataSource}
           subheader={<Trans id="button.download.data.all">Full dataset</Trans>}
@@ -353,13 +355,16 @@ const DownloadMenuItem = ({
       switch (fileFormat) {
         case "csv":
           const csv = await workbook.csv.writeBuffer();
-          saveAs(new Blob([csv], { type: "text/csv" }), `${fileName}.csv`);
+          saveAs(
+            new Blob(["\uFEFF" + csv], { type: "text/csv;charset=utf-8" }),
+            `${fileName}.csv`
+          );
           break;
         case "xlsx":
           const xlsx = await workbook.xlsx.writeBuffer();
           saveAs(
             new Blob([xlsx], {
-              type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+              type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;charset=utf-8",
             }),
             `${fileName}.xlsx`
           );

app/components/debug-panel/DebugPanel.tsx

@@ -125,8 +125,7 @@ const DebugConfigurator = () => {
             size="small"
             href={`${sparqlEditorUrl}#query=${encodeURIComponent(
               `#pragma describe.strategy cbd
-               #pragma join.hash off
-              
+
                DESCRIBE <${cube.iri}>`
             )}&requestMethod=POST`}
             target="_blank"

app/components/use-redirect-to-versioned-cube.spec.tsx

@@ -1,4 +1,4 @@
-import { renderHook } from "@testing-library/react-hooks";
+import { renderHook } from "@testing-library/react";
 import { NextRouter, useRouter } from "next/router";
 
 import { useLocale } from "@/locales/use-locale";

app/configurator/use-filter-changes.spec.tsx

@@ -1,4 +1,4 @@
-import { renderHook } from "@testing-library/react-hooks";
+import { renderHook } from "@testing-library/react";
 import merge from "lodash/merge";
 
 import { Filters } from "@/config-types";

app/domain/datasource/index.spec.ts

@@ -1,4 +1,4 @@
-import { act, renderHook } from "@testing-library/react-hooks";
+import { act, renderHook } from "@testing-library/react";
 import mittEmitter from "next/dist/shared/lib/mitt";
 import { SingletonRouter } from "next/router";
 

app/formatters.spec.ts

@@ -1,4 +1,4 @@
-import { renderHook } from "@testing-library/react-hooks";
+import { renderHook } from "@testing-library/react";
 
 import {
   getTimeIntervalFormattedSelectOptions,

app/graphql/context.tsx

@@ -10,9 +10,9 @@ import { LRUCache } from "typescript-lru-cache";
 import { SPARQL_GEO_ENDPOINT } from "@/domain/env";
 import { Awaited } from "@/domain/types";
 import { Timings } from "@/gql-flamegraph/resolvers";
-import { createSource } from "@/rdf/create-source";
+import { createSource, pragmas } from "@/rdf/create-source";
 import { ExtendedCube } from "@/rdf/extended-cube";
-import { timed, TimingCallback } from "@/utils/timed";
+import { TimingCallback, timed } from "@/utils/timed";
 
 import { createCubeDimensionValuesLoader } from "../rdf/queries";
 import {
@@ -27,7 +27,7 @@ import { RequestQueryMeta } from "./query-meta";
 export const MAX_BATCH_SIZE = 500;
 
 export const getRawCube = async (sparqlClient: ParsingClient, iri: string) => {
-  const source = createSource(sparqlClient);
+  const source = createSource(sparqlClient, pragmas);
   const cube = new ExtendedCube({
     parent: source,
     term: rdf.namedNode(iri),

app/package.json

@@ -40,7 +40,7 @@
     "@prisma/client": "^4.5.0",
     "@rdfjs/namespace": "^1.1.0",
     "@reach/auto-id": "^0.15.0",
-    "@sentry/nextjs": "^7.46.0",
+    "@sentry/nextjs": "^7.7.0",
     "@tpluscode/rdf-ns-builders": "2.0.1",
     "@tpluscode/rdf-string": "^0.2.26",
     "@tpluscode/sparql-builder": "^0.3.24",
@@ -145,8 +145,7 @@
     "@playwright-testing-library/test": "^4.5.0",
     "@playwright/test": "^1.32.1",
     "@svgr/cli": "^5.5.0",
-    "@testing-library/react": "^12.1.2",
-    "@testing-library/react-hooks": "^7.0.2",
+    "@testing-library/react": "^14.1.2",
     "@types/autosuggest-highlight": "^3.2.0",
     "@types/clownface": "^1.0.3",
     "@types/cors": "^2.8.8",

app/pages/api/graphql.ts

@@ -4,8 +4,9 @@ import configureCors from "cors";
 import "global-agent/bootstrap";
 import { NextApiRequest, NextApiResponse } from "next";
 
+import { SentryPlugin } from "@/graphql/apollo-sentry-plugin";
+
 import { setupFlamegraph } from "../../gql-flamegraph/resolvers";
-import { SentryPlugin } from "../../graphql/apollo-sentry-plugin";
 import { createContext, VisualizeGraphQLContext } from "../../graphql/context";
 import { resolvers } from "../../graphql/resolvers";
 import typeDefs from "../../graphql/schema.graphql";

app/rdf/create-source.ts

@@ -6,11 +6,14 @@ export const pragmas = `#pragma describe.strategy cbd
 #pragma join.hash off
 `;
 
-export const createSource = (sparqlClient: ParsingClient) => {
+export const createSource = (
+  sparqlClient: ParsingClient,
+  queryPrefix?: string
+) => {
   return new Source({
     client: sparqlClient,
     queryOperation: "postUrlencoded",
-    queryPrefix: pragmas,
+    queryPrefix,
     sourceGraph: rdf.defaultGraph(),
   });
 };

app/rdf/queries.ts

@@ -9,7 +9,7 @@ import { LRUCache } from "typescript-lru-cache";
 
 import { PromiseValue, truthy } from "@/domain/types";
 import { DataCubeComponentFilter } from "@/graphql/resolver-types";
-import { pragmas } from "@/rdf/create-source";
+import { createSource, pragmas } from "@/rdf/create-source";
 import { ExtendedCube } from "@/rdf/extended-cube";
 
 import { FilterValueMulti, Filters } from "../configurator";
@@ -610,6 +610,14 @@ export const getCubeObservations = async ({
     filters: observationFilters,
   });
 
+  // In order to fix an error with cartesian products introduced in preview query
+  // when using #pragma join.hash off, we need to have a clean source without
+  // decorating the sparql client. However we still need to keep the pragmas
+  // for the full query, to vastly improve performance.
+  observationsView.getMainSource = preview
+    ? () => createSource(sparqlClient)
+    : cubeView.getMainSource;
+
   const { query, observationsRaw } = await fetchViewObservations({
     preview,
     limit,

app/rdf/query-cube-metadata.ts

@@ -140,7 +140,10 @@ export const getCubeMetadata = async (
       ?contactPoint ${ns.vcard.hasEmail} ?contactPointEmail .
     }
     OPTIONAL { ?iri ${ns.dcterms.publisher} ?publisher . }
-    OPTIONAL { ?iri ${ns.dcat.landingPage} ?landingPage . }
+    ${buildLocalizedSubQuery("iri", "dcat:landingPage", "landingPage", {
+      locale,
+      fallbackToNonLocalized: true,
+    })}
     OPTIONAL { ?iri ${ns.schema.expires} ?expires . }
     OPTIONAL { ?iri ${ns.schema.workExample} ?workExample . }
   `.GROUP().BY`?iri`.THEN.BY`?identifier`.THEN.BY`?title`.THEN.BY`?description`

app/rdf/query-literals.ts

@@ -1,6 +1,6 @@
 import { SELECT, sparql } from "@tpluscode/sparql-builder";
 import uniqBy from "lodash/uniqBy";
-import { NamedNode, Literal } from "rdf-js";
+import { Literal, NamedNode } from "rdf-js";
 import ParsingClient from "sparql-http-client/ParsingClient";
 import { LRUCache } from "typescript-lru-cache";
 

app/rdf/query-utils.spec.ts

@@ -0,0 +1,67 @@
+import { buildLocalizedSubQuery } from "./query-utils";
+
+describe("buildLocalizedSubQuery", () => {
+  it("should build a subquery with the given locale", () => {
+    const subQuery = buildLocalizedSubQuery("s", "p", "o", {
+      locale: "it",
+    });
+    expect(subQuery).toEqual(
+      // it locale must be first!
+      `OPTIONAL {
+  ?s p ?o_it .
+  FILTER(LANG(?o_it) = "it")
+}
+OPTIONAL {
+  ?s p ?o_de .
+  FILTER(LANG(?o_de) = "de")
+}
+OPTIONAL {
+  ?s p ?o_fr .
+  FILTER(LANG(?o_fr) = "fr")
+}
+OPTIONAL {
+  ?s p ?o_en .
+  FILTER(LANG(?o_en) = "en")
+}
+OPTIONAL {
+  ?s p ?o_ .
+  FILTER(LANG(?o_) = "")
+}
+BIND(COALESCE(?o_it, ?o_de, ?o_fr, ?o_en, ?o_) AS ?o)`
+    );
+  });
+
+  it("should build a subquery with the given locale, falling back to non-localized property", () => {
+    const subQuery = buildLocalizedSubQuery("s", "p", "o", {
+      locale: "en",
+      fallbackToNonLocalized: true,
+    });
+    expect(subQuery).toEqual(
+      // en locale must be first!
+      `OPTIONAL {
+  ?s p ?o_en .
+  FILTER(LANG(?o_en) = "en")
+}
+OPTIONAL {
+  ?s p ?o_de .
+  FILTER(LANG(?o_de) = "de")
+}
+OPTIONAL {
+  ?s p ?o_fr .
+  FILTER(LANG(?o_fr) = "fr")
+}
+OPTIONAL {
+  ?s p ?o_it .
+  FILTER(LANG(?o_it) = "it")
+}
+OPTIONAL {
+  ?s p ?o_ .
+  FILTER(LANG(?o_) = "")
+}
+OPTIONAL {
+  ?s p ?o_raw .
+}
+BIND(COALESCE(?o_en, ?o_de, ?o_fr, ?o_it, ?o_, ?o_raw) AS ?o)`
+    );
+  });
+});

app/rdf/query-utils.ts

@@ -10,23 +10,34 @@ export const buildLocalizedSubQuery = (
   s: string,
   p: string,
   o: string,
-  { locale }: { locale: string }
+  {
+    locale,
+    fallbackToNonLocalized,
+  }: {
+    locale: string;
+    fallbackToNonLocalized?: boolean;
+  }
 ) => {
   // Include the empty locale as well.
   const locales = getOrderedLocales(locale).concat("");
 
   return `${locales
     .map(
       (locale) => `OPTIONAL {
-          ?${s} ${p} ?${o}_${locale} .
-          FILTER(LANG(?${o}_${locale}) = "${locale}")
-        }`
+  ?${s} ${p} ?${o}_${locale} .
+  FILTER(LANG(?${o}_${locale}) = "${locale}")
+}`
     )
-    .join("\n")}
-      BIND(COALESCE(${locales
-        .map((locale) => `?${o}_${locale}`)
-        .join(", ")}) AS ?${o})
-  `;
+    .join("\n")}${
+    fallbackToNonLocalized
+      ? `\nOPTIONAL {
+  ?${s} ${p} ?${o}_raw .
+}`
+      : ""
+  }
+BIND(COALESCE(${locales.map((locale) => `?${o}_${locale}`).join(", ")}${
+    fallbackToNonLocalized ? `, ?${o}_raw` : ``
+  }) AS ?${o})`;
 };
 
 const getOrderedLocales = (locale: string) => {

app/sentry.client.config.js

@@ -11,7 +11,7 @@ if (process.env.NODE_ENV !== "development") {
     dsn: SENTRY_DSN,
     environment: SENTRY_ENV,
     release: `visualization-tool@${BUILD_VERSION}`,
-    tracesSampleRate: 1.0,
+    tracesSampleRate: 0.1,
     ignoreErrors: [
       // The ResizeObserver error is actually not problematic
       // @see https://forum.sentry.io/t/resizeobserver-loop-limit-exceeded/8402

app/sentry.edge.config.js

@@ -11,6 +11,6 @@ if (process.env.NODE_ENV !== "development") {
     dsn: SENTRY_DSN,
     environment: SENTRY_ENV,
     release: `visualization-tool@${BUILD_VERSION}`,
-    tracesSampleRate: 1.0,
+    tracesSampleRate: 0.1,
   });
 }

app/sentry.server.config.js

@@ -11,7 +11,7 @@ if (process.env.NODE_ENV !== "development") {
     dsn: SENTRY_DSN,
     environment: SENTRY_ENV,
     release: `visualization-tool@${BUILD_VERSION}`,
-    tracesSampleRate: 1.0,
+    tracesSampleRate: 0.1,
     instrumenter: {
       patch: (mod, path, logger) => {
         // Ignore auth calls to prevent 405 Keycloak errors.