fix: Duplicated hierarchies

CHANGELOG.md

@@ -8,10 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 You can also check the [release page](https://github.com/visualize-admin/visualization-tool/releases)
 
 ## Unreleased
+
 - Features
   - Match drag and drop behavior for table chart and filter panel
 - Fixes
   - Remove ErrorWhisker and Tooltip when error is null in Column charts
+  - We now only fetch hierarchies defined in cube's shape
+  - Hierarchy names are now correctly retrieved
+- Performance
+  - We no longer fetch shape when initalizing the cube, as we might need to re-fetch it again if a newer cube is required
 
 # [3.24.0] - 2023-11-08
 

app/graphql/context.tsx

@@ -2,6 +2,7 @@ import { IncomingMessage } from "http";
 
 import DataLoader from "dataloader";
 import { GraphQLResolveInfo } from "graphql";
+import rdf from "rdf-ext";
 import StreamClient from "sparql-http-client";
 import ParsingClient from "sparql-http-client/ParsingClient";
 import { LRUCache } from "typescript-lru-cache";
@@ -10,6 +11,7 @@ import { SPARQL_GEO_ENDPOINT } from "@/domain/env";
 import { Awaited } from "@/domain/types";
 import { Timings } from "@/gql-flamegraph/resolvers";
 import { createSource } from "@/rdf/create-source";
+import { ExtendedCube } from "@/rdf/extended-cube";
 import { timed, TimingCallback } from "@/utils/timed";
 
 import { createCubeDimensionValuesLoader } from "../rdf/queries";
@@ -26,7 +28,15 @@ export const MAX_BATCH_SIZE = 500;
 
 export const getRawCube = async (sourceUrl: string, iri: string) => {
   const source = createSource({ endpointUrl: sourceUrl });
-  return await source.cube(iri);
+  const cube = new ExtendedCube({
+    parent: source,
+    term: rdf.namedNode(iri),
+    source,
+  });
+  // Don't fetch shape yet, as we might need to fetch newer cube.
+  await cube.fetchCube();
+
+  return cube;
 };
 
 // const cachedGetRawCube = cachedWithTTL(

app/graphql/resolvers.spec.ts

@@ -3,6 +3,7 @@ import { IncomingMessage } from "http";
 import { GraphQLResolveInfo } from "graphql";
 
 import { createSource as createSource_ } from "@/rdf/create-source";
+import { ExtendedCube as ExtendedCube_ } from "@/rdf/extended-cube";
 import { getCubeObservations as getCubeObservations_ } from "@/rdf/queries";
 import { unversionObservation as unversionObservation_ } from "@/rdf/query-dimension-values";
 
@@ -16,6 +17,10 @@ const createSource = createSource_ as unknown as jest.Mock<
   typeof createSource_
 >;
 
+const ExtendedCube = ExtendedCube_ as unknown as jest.Mock<
+  typeof ExtendedCube_
+>;
+
 const unversionObservation = unversionObservation_ as unknown as jest.Mock<
   typeof unversionObservation_
 >;
@@ -27,12 +32,18 @@ jest.mock("../rdf/query-search", () => ({}));
 jest.mock("../rdf/queries", () => ({
   getCubeObservations: jest.fn(),
   createCubeDimensionValuesLoader: () => async () => [],
+  getLatestCube: () => ({
+    fetchShape: () => ({}),
+  }),
 }));
 jest.mock("../rdf/create-source", () => ({
   createSource: jest.fn(),
 }));
 jest.mock("../rdf/query-hierarchies", () => ({}));
 jest.mock("../rdf/parse", () => ({}));
+jest.mock("../rdf/extended-cube", () => ({
+  ExtendedCube: jest.fn(),
+}));
 
 jest.mock("@rdf-esm/data-model", () => ({}));
 jest.mock("@rdf-esm/term-map", () => ({}));
@@ -45,6 +56,7 @@ describe("possible filters", () => {
   beforeEach(() => {
     getCubeObservations.mockReset();
   });
+
   it("should try to find an observation given possible filters, relaxing fitlers from the bottom", async () => {
     // @ts-ignore
     getCubeObservations.mockImplementation(async ({ filters }) => {
@@ -74,6 +86,11 @@ describe("possible filters", () => {
     createSource.mockImplementation(() => ({
       cube: () => ({}),
     }));
+    // @ts-ignore
+    ExtendedCube.mockImplementation(() => ({
+      fetchCube: () => ({}),
+      fetchShape: () => ({}),
+    }));
 
     const res = await Query?.possibleFilters?.(
       {},
@@ -94,9 +111,18 @@ describe("possible filters", () => {
         },
       } as unknown as GraphQLResolveInfo
     );
+
     expect(res).toEqual([
-      { iri: "https://fake-dimension-iri-1", type: "single", value: 1 },
-      { iri: "https://fake-dimension-iri-2", type: "single", value: 3 },
+      {
+        iri: "https://fake-dimension-iri-1",
+        type: "single",
+        value: 1,
+      },
+      {
+        iri: "https://fake-dimension-iri-2",
+        type: "single",
+        value: 3,
+      },
     ]);
     expect(getCubeObservations).toHaveBeenCalledTimes(2);
   });

app/graphql/resolvers/rdf.ts

@@ -13,11 +13,13 @@ import {
   Resolvers,
   SearchCubeResultOrder,
 } from "@/graphql/resolver-types";
+import { defaultLocale } from "@/locales/locales";
+import { parseCube } from "@/rdf/parse";
 import {
   createCubeDimensionValuesLoader,
   getCubeDimensions,
   getCubeObservations,
-  getResolvedCube,
+  getLatestCube,
 } from "@/rdf/queries";
 import { unversionObservation } from "@/rdf/query-dimension-values";
 import { queryHierarchy } from "@/rdf/query-hierarchies";
@@ -73,21 +75,27 @@ export const searchCubes: NonNullable<QueryResolvers["searchCubes"]> = async (
 };
 
 export const dataCubeByIri: NonNullable<QueryResolvers["dataCubeByIri"]> =
-  async (_, { iri, locale, latest }, { setup }, info) => {
+  async (_, { iri, locale, latest = true }, { setup }, info) => {
     const { loaders } = await setup(info);
-    const cube = await loaders.cube.load(iri);
+    const rawCube = await loaders.cube.load(iri);
 
-    if (!cube) {
+    if (!rawCube) {
       throw new Error("Cube not found");
     }
 
-    return getResolvedCube({ cube, locale: locale || "de", latest });
+    const cube = latest ? await getLatestCube(rawCube) : rawCube;
+    await cube.fetchShape();
+
+    return parseCube({ cube, locale: locale ?? defaultLocale });
   };
 
 export const possibleFilters: NonNullable<QueryResolvers["possibleFilters"]> =
   async (_, { iri, filters }, { setup }, info) => {
     const { sparqlClient, loaders, cache } = await setup(info);
-    const cube = await loaders.cube.load(iri);
+    const rawCube = await loaders.cube.load(iri);
+    // Currently we always default to the latest cube.
+    const cube = await getLatestCube(rawCube);
+    await cube.fetchShape();
 
     if (!cube) {
       return [];
@@ -114,16 +122,15 @@ export const possibleFilters: NonNullable<QueryResolvers["possibleFilters"]> =
 
       const unversioned = await unversionObservation({
         observation: obs[0],
-        cube: cube,
+        cube,
         sparqlClient,
       });
-      const result = Object.keys(filters).map((f) => ({
+
+      return Object.keys(filters).map((f) => ({
         iri: f,
         type: "single",
         value: unversioned[f],
       }));
-
-      return result;
     }
 
     return [];
@@ -251,15 +258,13 @@ export const hierarchy: NonNullable<DimensionResolvers["hierarchy"]> = async (
     throw new Error("Could not find cube");
   }
 
-  const res = await queryHierarchy(
+  return await queryHierarchy(
     rdimension,
     locale,
     sparqlClient,
     sparqlClientStream,
     cache
   );
-
-  return res;
 };
 
 export const dimensionValues: NonNullable<

app/graphql/shared-types.ts

@@ -1,6 +1,8 @@
-import { Cube, CubeDimension } from "rdf-cube-view-query";
+import { CubeDimension } from "rdf-cube-view-query";
 import { Literal, NamedNode } from "rdf-js";
 
+import { ExtendedCube } from "@/rdf/extended-cube";
+
 import { Observation } from "../domain/data";
 
 import { RelatedDimension } from "./query-hooks";
@@ -15,7 +17,7 @@ import {
 /** Types shared by graphql-codegen and resolver code */
 
 export type ResolvedDataCube = {
-  cube: Cube;
+  cube: ExtendedCube;
   locale: string;
   data: {
     iri: string;
@@ -42,7 +44,7 @@ export type ResolvedDataCube = {
 };
 
 export type ResolvedDimension = {
-  cube: Cube;
+  cube: ExtendedCube;
   dimension: CubeDimension;
   locale: string;
   data: {
@@ -73,7 +75,7 @@ export type ResolvedDimension = {
 export type ResolvedMeasure = ResolvedDimension;
 
 export type ResolvedObservationsQuery = {
-  cube: Cube;
+  cube: ExtendedCube;
   locale: string;
 
   data: {

app/rdf/cube-filters.ts

@@ -1,9 +1,9 @@
-import { Cube } from "rdf-cube-view-query";
 import rdf from "rdf-ext";
 import { NamedNode } from "rdf-js";
 
 import { truthy } from "@/domain/types";
 import { SearchCubeFilter } from "@/graphql/resolver-types";
+import { ExtendedCube } from "@/rdf/extended-cube";
 import * as ns from "@/rdf/namespace";
 
 import isAttrEqual from "../utils/is-attr-equal";
@@ -22,7 +22,7 @@ export const makeInQueryFilter = (
   filters: SearchCubeFilter[]
 ) => {
   return filters.length > 0
-    ? Cube.filter.in(
+    ? ExtendedCube.filter.in(
         predicate,
         filters.map((x) => rdf.namedNode(x.value))
       )
@@ -51,11 +51,11 @@ export const makeCubeFilters = ({
 
   const res = [
     // Cubes that have a newer version published have a schema.org/expires property; Only show cubes that don't have it
-    Cube.filter.noExpires(),
+    ExtendedCube.filter.noExpires(),
     isVisualizeCubeFilter,
     includeDrafts
       ? null
-      : Cube.filter.status([
+      : ExtendedCube.filter.status([
           ns.adminVocabulary("CreativeWorkStatus/Published"),
         ]),
     themeQueryFilter,

app/rdf/extended-cube.ts

@@ -0,0 +1,29 @@
+import clownface, { AnyPointer } from "clownface";
+import { Cube, CubeOptions } from "rdf-cube-view-query";
+import rdf from "rdf-ext";
+import DatasetExt from "rdf-ext/lib/Dataset";
+
+export class ExtendedCube extends Cube {
+  private shapeDataset: DatasetExt;
+  public shapePtr: AnyPointer;
+
+  constructor(options: CubeOptions) {
+    const { term = rdf.blankNode(), graph } = options;
+    super(options);
+    this.shapeDataset = rdf.dataset();
+    this.shapePtr = clownface({
+      term,
+      dataset: this.shapeDataset,
+      graph,
+    });
+  }
+
+  async fetchShape() {
+    const shapeData = await this.source.client.query.construct(
+      this.shapeQuery()
+    );
+    this.dataset.addAll(shapeData);
+    this.shapeDataset.addAll(shapeData);
+    this.quads = [...this.quads, ...shapeData];
+  }
+}

app/rdf/parse.ts

@@ -10,11 +10,12 @@ import {
   timeWeek,
   timeYear,
 } from "d3";
-import { Cube, CubeDimension } from "rdf-cube-view-query";
+import { CubeDimension } from "rdf-cube-view-query";
 import { NamedNode, Term } from "rdf-js";
 
 import { truthy } from "@/domain/types";
 import { ScaleType } from "@/graphql/query-hooks";
+import { ExtendedCube } from "@/rdf/extended-cube";
 
 import { DataCubePublicationStatus, TimeUnit } from "../graphql/resolver-types";
 import { ResolvedDataCube, ResolvedDimension } from "../graphql/shared-types";
@@ -29,18 +30,18 @@ export const getQueryLocales = (locale: string): string[] => [
   "",
 ];
 
-export const isCubePublished = (cube: Cube): boolean =>
+export const isCubePublished = (cube: ExtendedCube): boolean =>
   cube
     .out(ns.schema.creativeWorkStatus)
     .terms.some((t) =>
       t.equals(ns.adminVocabulary("CreativeWorkStatus/Published"))
     );
 
-export const parseVersionHistory = (cube: Cube) => {
+export const parseVersionHistory = (cube: ExtendedCube) => {
   return cube.in(ns.schema.hasPart)?.value;
 };
 
-export const parseIri = (cube: Cube) => {
+export const parseIri = (cube: ExtendedCube) => {
   return cube.term?.value ?? "[NO IRI]";
 };
 
@@ -53,7 +54,7 @@ export const parseCube = ({
   cube,
   locale,
 }: {
-  cube: Cube;
+  cube: ExtendedCube;
   locale: string;
 }): ResolvedDataCube => {
   const outOpts = { language: getQueryLocales(locale) };
@@ -206,7 +207,7 @@ export const parseCubeDimension = ({
   units,
 }: {
   dim: CubeDimension;
-  cube: Cube;
+  cube: ExtendedCube;
   locale: string;
   units?: Map<
     string,