Merge pull request #1849 from visualize-admin/feat/finalize-merging-o…

…f-cubes-2

feat: Introduce component ids

.github/workflows/e2e.yml

@@ -6,6 +6,11 @@ jobs:
     # only runs this job on successful deploy
     if: github.event.deployment_status.state == 'success'
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -20,10 +25,20 @@ jobs:
       - name: Install Playwright Browsers
         run: npx playwright install --with-deps chromium
       - name: Run Playwright tests
-        run: yarn e2e -- --grep-invert @noci
+        run:
+          yarn e2e -- --grep-invert @noci --shard=${{ matrix.shardIndex }}/${{
+          matrix.shardTotal }}
         env:
           E2E_BASE_URL: ${{ github.event.deployment_status.target_url }}
           PERCY_TOKEN: ${{ secrets.PERCY_TOKEN }}
+          E2E_HAR: ${{ secrets.E2E_HAR }}
+      - name: Upload blob report to GitHub Actions Artifacts
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: blob-report-${{ matrix.shardIndex }}
+          path: blob-report
+          retention-days: 1
       - name: Upload E2E artifacts to job
         uses: actions/upload-artifact@v3
         if: failure()
@@ -32,4 +47,36 @@ jobs:
           path: |
             e2e-screenshots
             playwright-report
-      # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#job-status-check-functions
+  merge-reports:
+    if: ${{ !cancelled() }}
+    needs: [e2e]
+
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # fetch all commits so we can find the branch
+          fetch-depth: 0
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+          cache: "yarn"
+      - name: Install dependencies
+        run: yarn install --immutable
+
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+
+      - name: Merge into HTML Report
+        run: npx playwright merge-reports --reporter html ./all-blob-reports
+
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report--attempt-${{ github.run_attempt }}
+          path: playwright-report
+          retention-days: 14

.gitignore

@@ -17,6 +17,7 @@ node_modules
 
 __diff_output__
 
+screenshots
 e2e-screenshots
 playwright-report
 

CHANGELOG.md

@@ -11,7 +11,19 @@ You can also check the
 
 # Unreleased
 
-Nothing yet.
+- Features
+  - Allowed previewing of charts in states different than CONFIGURING_CHART
+    (iframe `/preview` page)
+- Fixes
+  - Introduced a `componentId` concept which makes the dimensions and measures
+    unique by adding an unversioned cube iri to the unversioned component iri on
+    the server-side. With this we correctly handle multiple dimensions with the
+    same iri that come from different cubes (when merging cubes)
+  - Map legend is now correctly updated (in some cases it was rendered
+    incorrectly on the initial render)
+- Maintenance
+  - Added a way to do local visual regression testing of charts between
+    different branches
 
 # [4.9.5] - 2024-11-18
 

README.md

@@ -17,18 +17,19 @@
   - 4.3. [Docker (anywhere)](#Dockeranywhere)
 - 5. [Developing GitHub Actions](#DevelopingGitHubActions)
 - 6. [E2E tests](#E2Etests)
-- 7. [GraphQL performance tests](#GraphQLperformancetests)
-  - 7.1. [Automation](#Automation)
-  - 7.2. [How to add or modify the tests](#Howtoaddormodifythetests)
-- 8. [Load tests](#Loadtests)
-  - 8.1. [Automation](#Automation-1)
-  - 8.2. [Local setup](#Localsetup)
-  - 8.3. [Running the tests locally](#Runningthetestslocally)
-  - 8.4.
+- 7. [Visual regression tests](#Visualregressiontests)
+- 8. [GraphQL performance tests](#GraphQLperformancetests)
+  - 8.1. [Automation](#Automation)
+  - 8.2. [How to add or modify the tests](#Howtoaddormodifythetests)
+- 9. [Load tests](#Loadtests)
+  - 9.1. [Automation](#Automation-1)
+  - 9.2. [Local setup](#Localsetup)
+  - 9.3. [Running the tests locally](#Runningthetestslocally)
+  - 9.4.
     [Recording the tests using Playwright](#RecordingthetestsusingPlaywright)
-- 9. [Authentication](#Authentication)
-  - 9.1. [Locally](#Locally)
-- 10. [Translations](#Translations)
+- 10. [Authentication](#Authentication)
+  - 10.1. [Locally](#Locally)
+- 11. [Translations](#Translations)
 
 <!-- vscode-markdown-toc-config
 	numbering=true
@@ -254,35 +255,46 @@ made with Percy and sent directly to their cloud service for diffing.
 A special [test page](http://localhost:3000/en/__test) shows all the charts that
 are screenshotted. Those charts configurations are kept in the repository.
 
-## 7. <a name='GraphQLperformancetests'></a>GraphQL performance tests
+## 7. <a name='Visualregressiontests'></a>Visual regression tests
+
+It's sometimes useful to run visual regression tests, especially when modifying
+chart configurator or chart config schemas. To make sure that the changes don't
+break the existing charts, we implemented a way to do a baseline vs. comparison
+tests.
+
+To run the tests, you should check out the instruction in
+[e2e/all-charts.spec.ts](https://github.com/visualize-admin/visualization-tool/blob/main/e2e/all-charts.spec.ts)
+file.
+
+## 8. <a name='GraphQLperformancetests'></a>GraphQL performance tests
 
 The project uses a combination of [k6](https://k6.io) and
 [Grafana](https://grafana.com) with
 [Prometheus](https://k6.io/docs/results-output/real-time/prometheus-remote-write/)
 for GraphQL performance testing.
 
-### 7.1. <a name='Automation'></a>Automation
+### 8.1. <a name='Automation'></a>Automation
 
 To ensure constant monitoring of the performance of selected GraphQL queries,
 the performance tests are run
 [once an hour](https://github.com/visualize-admin/visualization-tool/blob/main/.github/workflows/performance-tests.yml)
 against each environment of the application. The results are then sent to the
 governmental Grafana dashboards.
 
-### 7.2. <a name='Howtoaddormodifythetests'></a>How to add or modify the tests
+### 8.2. <a name='Howtoaddormodifythetests'></a>How to add or modify the tests
 
 To add or modify the performance tests, go to the
 [k6/performance-tests](https://github.com/visualize-admin/visualization-tool/tree/main/k6/performance-tests)
 folder. The GitHub Action is generated by running the `yarn run github:codegen`
 command; be sure to run it after modifying the
-[generate-github-action.js](`https://github.com/visualize-admin/visualization-tool/blob/main/k6/performance-tests/generate-github-action.js`)
+[generate-github-actions.mjs](https://github.com/visualize-admin/visualization-tool/blob/main/k6/performance-tests/generate-github-actions.mjs)
 file.
 
-## 8. <a name='Loadtests'></a>Load tests
+## 9. <a name='Loadtests'></a>Load tests
 
 The project uses [k6](https://k6.io) for load testing.
 
-### 8.1. <a name='Automation-1'></a>Automation
+### 9.1. <a name='Automation-1'></a>Automation
 
 There is a
 [dedicated GitHub Action](https://github.com/visualize-admin/visualization-tool/actions/workflows/manual-load-test.yml)
@@ -291,13 +303,13 @@ the results by going to Actions section in GitHub and checking the summary
 results. They are also visible in the cloud (k6.io), if you enable the cloud
 option.
 
-### 8.2. <a name='Localsetup'></a>Local setup
+### 9.2. <a name='Localsetup'></a>Local setup
 
 In order to run the tests locally, follow the
 [documentation](https://k6.io/docs/get-started/installation/) to install `k6` on
 your machine.
 
-### 8.3. <a name='Runningthetestslocally'></a>Running the tests locally
+### 9.3. <a name='Runningthetestslocally'></a>Running the tests locally
 
 You might want to run the script locally, for example to be able to bypass the
 cloud limitations of k6 (e.g. max number of VUs bigger than 50). To run a given
@@ -317,7 +329,7 @@ to point to the proper environment and
 `--env ENABLE_GQL_SERVER_SIDE_CACHE=(true | false)` to control whether to use
 GQL server-side caching.
 
-### 8.4. <a name='RecordingthetestsusingPlaywright'></a>Recording the tests using Playwright
+### 9.4. <a name='RecordingthetestsusingPlaywright'></a>Recording the tests using Playwright
 
 While some tests are written manually, other tests come from HAR recordings that
 span a broad set of actions. In order to record a HAR file and convert it into
@@ -340,13 +352,13 @@ parameters for the test.
 > ⚠️ You might want to remove requests to Google afterwards manually, to remove
 > false-positives of blocked requests.
 
-## 9. <a name='Authentication'></a>Authentication
+## 10. <a name='Authentication'></a>Authentication
 
 Authentication is provided by the Swiss federal government's eIAM through ADFS.
 We use Next-auth to integrate our application with it, through a
 [custom Provider](app/auth-providers/adfs.ts).
 
-### 9.1. <a name='Locally'></a>Locally
+### 10.1. <a name='Locally'></a>Locally
 
 You can use the ref eIAM. ADFS environment variables should be configured in
 your `.env.local` file. You'll find those secret variables in our shared
@@ -355,7 +367,7 @@ your `.env.local` file. You'll find those secret variables in our shared
 Make sure to set the `NEXTAUTH_URL` environment variable to
 `https://localhost:3000` and run the dev server with `yarn dev:ssl`.
 
-## 10. <a name='Translations'></a>Translations
+## 11. <a name='Translations'></a>Translations
 
 Translations are managed via [Accent](https://accent.interactivethings.io).
 Right now, you need to manually pull and push the translations. The process

app/browse/datatable.tsx

@@ -12,10 +12,10 @@ import {
   TooltipProps,
 } from "@mui/material";
 import { ascending, descending } from "d3-array";
-import { useMemo, useRef, useState } from "react";
+import { useCallback, useMemo, useRef, useState } from "react";
 
 import {
-  extractChartConfigComponentIris,
+  extractChartConfigComponentIds,
   useQueryFilters,
 } from "@/charts/shared/chart-helpers";
 import { Loading } from "@/components/hint";
@@ -92,62 +92,64 @@ export const PreviewTable = ({
     const valuesIndex = uniqueMapBy(sortBy.values, (d) => d.label);
     const convert =
       isNumericalMeasure(sortBy) || sortBy.isNumerical
-        ? (value: string) => +value
-        : (value: string) => valuesIndex.get(value)?.position ?? value;
+        ? (v: string) => +v
+        : (v: string) => valuesIndex.get(v)?.position ?? v;
 
     return [...observations].sort((a, b) =>
-      compare(
-        convert(a[sortBy.iri] as string),
-        convert(b[sortBy.iri] as string)
-      )
+      compare(convert(a[sortBy.id] as string), convert(b[sortBy.id] as string))
     );
   }, [observations, sortBy, sortDirection]);
 
-  const tooltipContainerRef = useRef(null);
+  const tooltipContainerRef = useRef<HTMLDivElement>(null);
+  const tooltipProps = useMemo(() => {
+    return {
+      PopperProps: {
+        // Tooltip contained inside the table so as not to overflow when table is scrolled
+        container: tooltipContainerRef.current,
+      },
+    };
+  }, []);
 
-  // Tooltip contained inside the table so as not to overflow when table is scrolled
-  const tooltipProps = useMemo(
-    () => ({ PopperProps: { container: tooltipContainerRef.current } }),
-    []
+  const handleSort = useCallback(
+    (header: Component) => {
+      if (sortBy?.id === header.id) {
+        setSortDirection(sortDirection === "asc" ? "desc" : "asc");
+      } else {
+        setSortBy(header);
+        setSortDirection("asc");
+      }
+    },
+    [sortBy, sortDirection]
   );
 
   return (
     <div ref={tooltipContainerRef}>
       <Table>
         <caption style={{ display: "none" }}>{title}</caption>
-        <TableHead sx={{ position: "sticky", top: 0, background: "white" }}>
+        <TableHead
+          sx={{ position: "sticky", top: 0, background: "background.paper" }}
+        >
           <TableRow sx={{ borderBottom: "none" }}>
             {headers.map((header) => {
               return (
                 <TableCell
-                  key={header.iri}
+                  key={header.id}
                   component="th"
                   role="columnheader"
-                  onClick={() => {
-                    if (sortBy?.iri === header.iri) {
-                      setSortDirection(
-                        sortDirection === "asc" ? "desc" : "asc"
-                      );
-                    } else {
-                      setSortBy(header);
-                      setSortDirection("asc");
-                    }
-                  }}
+                  onClick={() => handleSort(header)}
                   sx={{
                     textAlign: isNumericalMeasure(header) ? "right" : "left",
                     borderBottom: "none",
                     whiteSpace: "nowrap",
                   }}
                 >
                   <TableSortLabel
-                    active={sortBy === undefined || sortBy.iri === header.iri}
-                    direction={sortBy === undefined ? "desc" : sortDirection}
+                    active={!sortBy || sortBy.id === header.id}
+                    direction={!sortBy ? "desc" : sortDirection}
                     IconComponent={SvgIcChevronDown}
                     sx={{
                       "&:hover > svg": {
-                        ...(sortBy === undefined
-                          ? { transform: "translateY(-15%)" }
-                          : {}),
+                        ...(!sortBy ? { transform: "translateY(-15%)" } : {}),
                       },
                     }}
                   >
@@ -168,18 +170,19 @@ export const PreviewTable = ({
             return (
               <TableRow key={i}>
                 {headers.map((header) => {
-                  const formatter = formatters[header.iri];
+                  const format = formatters[header.id];
+
                   return (
                     <TableCell
-                      key={header.iri}
+                      key={header.id}
                       component="td"
                       sx={{
                         textAlign: isNumericalMeasure(header)
                           ? "right"
                           : "left",
                       }}
                     >
-                      {formatter(obs[header.iri])}
+                      {format(obs[header.id])}
                     </TableCell>
                   );
                 })}
@@ -242,7 +245,7 @@ export const DataSetTable = ({
   sx?: SxProps<Theme>;
 }) => {
   const locale = useLocale();
-  const componentIris = extractChartConfigComponentIris({ chartConfig });
+  const componentIds = extractChartConfigComponentIds({ chartConfig });
   const commonQueryVariables = {
     sourceType: dataSource.type,
     sourceUrl: dataSource.url,
@@ -260,7 +263,7 @@ export const DataSetTable = ({
         ...commonQueryVariables,
         cubeFilters: chartConfig.cubes.map((cube) => ({
           iri: cube.iri,
-          componentIris,
+          componentIds,
           joinBy: cube.joinBy,
         })),
       },
@@ -278,7 +281,7 @@ export const DataSetTable = ({
   const queryFilters = useQueryFilters({
     chartConfig,
     dashboardFilters,
-    componentIris,
+    componentIds,
   });
   const [{ data: observationsData }] = useDataCubesObservationsQuery({
     variables: {

app/browser/context.tsx

@@ -297,7 +297,7 @@ export const BrowseStateProvider = ({
 export const useBrowseContext = () => {
   const ctx = useContext(BrowseContext);
   if (!ctx) {
-    throw new Error(
+    throw Error(
       "To be able useBrowseContext, you must wrap it into a BrowseStateProvider"
     );
   }