From 6143c45586abd9bd8c077600ed0641d5295b810b Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Wed, 25 Oct 2023 13:16:07 +0200 Subject: [PATCH 01/48] fix: Allow dataset tags to wrap --- app/browser/dataset-browse.tsx | 4 ++-- app/components/tag.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index a3573bfe7..303310bdb 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -939,7 +939,7 @@ export const DatasetResult = ({ )} - + {themes && showTags ? sortBy(themes, (t) => t.label).map((t) => ( ) : null} - + ); }; diff --git a/app/components/tag.tsx b/app/components/tag.tsx index fbc805eb0..4b1653375 100644 --- a/app/components/tag.tsx +++ b/app/components/tag.tsx @@ -1,4 +1,4 @@ -import { Typography, styled, BoxProps, TypographyProps } from "@mui/material"; +import { BoxProps, Typography, TypographyProps, styled } from "@mui/material"; import React from "react"; import { DataCubeOrganization, DataCubeTheme } from "@/graphql/resolver-types"; From c82f4212457745b5808560972bb026e844ec9e29 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Wed, 25 Oct 2023 13:39:41 +0200 Subject: [PATCH 02/48] perf: Do not query dataset counts ...as we can compute them dynamically. This commit also fixes incorrect dataset counts in some cases. --- app/browser/dataset-browse.tsx | 45 ++++++++++++------------ app/browser/use-dataset-count.tsx | 38 -------------------- app/graphql/queries/data-cubes.graphql | 21 ----------- app/graphql/query-hooks.ts | 48 -------------------------- app/graphql/resolver-types.ts | 27 --------------- app/graphql/resolvers/index.ts | 4 --- app/graphql/resolvers/rdf.ts | 33 ++---------------- app/graphql/resolvers/sql.ts | 5 --- app/graphql/schema.graphql | 13 ------- app/rdf/query-search.ts | 6 ++-- 10 files changed, 28 insertions(+), 212 deletions(-) delete mode 100644 app/browser/use-dataset-count.tsx diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index 303310bdb..567c68467 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -58,7 +58,6 @@ import { useBrowseContext, } from "./context"; import { BrowseFilter } from "./filters"; -import useDatasetCount from "./use-dataset-count"; export const SearchDatasetInput = ({ browseState, @@ -433,7 +432,7 @@ export const Subthemes = ({ }: { organization: DataCubeOrganization; filters: BrowseFilter[]; - counts: ReturnType; + counts: Record; }) => { const termsetIri = organizationIriToTermsetParentIri[organization.iri]; const { dataSource } = useDataSourceStore(); @@ -606,7 +605,7 @@ const NavSection = ({ export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { const { dataSource } = useDataSourceStore(); const locale = useLocale(); - const { filters, search, includeDrafts } = useBrowseContext(); + const { filters } = useBrowseContext(); const [{ data: allThemes }] = useThemesQuery({ variables: { sourceType: dataSource.type, @@ -622,29 +621,23 @@ export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { }, }); - const allCounts = useDatasetCount(filters, includeDrafts); - const resultsCounts = useMemo(() => { + const counts = useMemo(() => { if (!data?.dataCubes) { return {}; - } else { - const res = {} as Record; - for (const cube of data.dataCubes) { - const countables = [ - ...cube.dataCube.themes, - cube.dataCube.creator, - ].filter(truthy); - for (const item of countables) { - res[item.iri] = res[item.iri] || 0; - res[item.iri] += 1; - } + } + + const result: Record = {}; + + for (const { dataCube } of data.dataCubes) { + const countables = [...dataCube.themes, dataCube.creator].filter(truthy); + + for (const { iri } of countables) { + result[iri] = (result[iri] ?? 0) + 1; } - return res; } - }, [data]); - const total = Object.values(resultsCounts).reduce((acc, n) => acc + n, 0); - const counts = - search && search != "" && total > 0 ? resultsCounts : allCounts; + return result; + }, [data?.dataCubes]); const themeFilter = filters.find(isAttrEqual("__typename", "DataCubeTheme")); const orgFilter = filters.find( @@ -662,12 +655,15 @@ export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { if (!theme.label) { return false; } + if (!counts[theme.iri]) { return false; } + if (themeFilter && themeFilter !== theme) { return false; } + return true; }); @@ -675,12 +671,15 @@ export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { if (!org.label) { return false; } + if (!counts[org.iri] && orgFilter !== org) { return false; } + if (orgFilter && orgFilter !== org) { return false; } + return true; }); @@ -727,7 +726,9 @@ export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { } /> ) : null; + let navs = [themeNav, orgNav]; + if (filters[0]?.__typename === "DataCubeTheme") { navs = [themeNav, orgNav]; } else if (filters[0]?.__typename === "DataCubeOrganization") { @@ -745,7 +746,6 @@ export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { role="search" key={filters.length} > - {/* Theme tree */} {navs[0]} {navs[1]} @@ -762,6 +762,7 @@ export const DatasetResults = ({ query: UseQueryState; }) => { const { fetching, data, error } = query; + if (fetching) { return ( diff --git a/app/browser/use-dataset-count.tsx b/app/browser/use-dataset-count.tsx deleted file mode 100644 index fac5066dd..000000000 --- a/app/browser/use-dataset-count.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import { useMemo } from "react"; - -import { useDatasetCountQuery } from "@/graphql/query-hooks"; -import { useDataSourceStore } from "@/stores/data-source"; -import isAttrEqual from "@/utils/is-attr-equal"; - -import { BrowseFilter } from "./filters"; - -const countListToIndexedCount = (l: { count: number; iri: string }[]) => - Object.fromEntries(l.map((o) => [o.iri, o.count])); - -const useDatasetCount = ( - filters: BrowseFilter[], - includeDrafts: boolean -): Record => { - const { dataSource } = useDataSourceStore(); - const [{ data: datasetCounts }] = useDatasetCountQuery({ - variables: { - sourceType: dataSource.type, - sourceUrl: dataSource.url, - includeDrafts, - theme: filters.find(isAttrEqual("__typename", "DataCubeTheme"))?.iri, - organization: filters.find( - isAttrEqual("__typename", "DataCubeOrganization") - )?.iri, - }, - }); - - return useMemo( - () => - datasetCounts?.datasetcount - ? countListToIndexedCount(datasetCounts?.datasetcount) - : {}, - [datasetCounts] - ); -}; - -export default useDatasetCount; diff --git a/app/graphql/queries/data-cubes.graphql b/app/graphql/queries/data-cubes.graphql index b60f068c9..e55819831 100644 --- a/app/graphql/queries/data-cubes.graphql +++ b/app/graphql/queries/data-cubes.graphql @@ -475,24 +475,3 @@ query DimensionHierarchy( } } } - -query DatasetCount( - $sourceType: String! - $sourceUrl: String! - $theme: String - $organization: String - $subtheme: String - $includeDrafts: Boolean -) { - datasetcount( - sourceType: $sourceType - sourceUrl: $sourceUrl - theme: $theme - organization: $organization - subtheme: $subtheme - includeDrafts: $includeDrafts - ) { - count - iri - } -} diff --git a/app/graphql/query-hooks.ts b/app/graphql/query-hooks.ts index 12560b4f8..1d6bbdc8d 100644 --- a/app/graphql/query-hooks.ts +++ b/app/graphql/query-hooks.ts @@ -118,12 +118,6 @@ export type DataCubeTheme = { label?: Maybe; }; -export type DatasetCount = { - __typename: 'DatasetCount'; - iri: Scalars['String']; - count: Scalars['Int']; -}; - export type Dimension = { iri: Scalars['String']; label: Scalars['String']; @@ -394,7 +388,6 @@ export type Query = { themes: Array; subthemes: Array; organizations: Array; - datasetcount?: Maybe>; }; @@ -451,16 +444,6 @@ export type QueryOrganizationsArgs = { }; -export type QueryDatasetcountArgs = { - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - theme?: Maybe; - organization?: Maybe; - subtheme?: Maybe; - includeDrafts?: Maybe; -}; - - export type RelatedDimension = { __typename: 'RelatedDimension'; type: Scalars['String']; @@ -1146,18 +1129,6 @@ export type DimensionHierarchyQuery = { __typename: 'Query', dataCubeByIri?: May & HierarchyMetadata_TemporalOrdinalDimension_Fragment )> }> }; -export type DatasetCountQueryVariables = Exact<{ - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - theme?: Maybe; - organization?: Maybe; - subtheme?: Maybe; - includeDrafts?: Maybe; -}>; - - -export type DatasetCountQuery = { __typename: 'Query', datasetcount?: Maybe> }; - export const DimensionMetadataFragmentDoc = gql` fragment dimensionMetadata on Dimension { iri @@ -1615,23 +1586,4 @@ export const DimensionHierarchyDocument = gql` export function useDimensionHierarchyQuery(options: Omit, 'query'> = {}) { return Urql.useQuery({ query: DimensionHierarchyDocument, ...options }); -}; -export const DatasetCountDocument = gql` - query DatasetCount($sourceType: String!, $sourceUrl: String!, $theme: String, $organization: String, $subtheme: String, $includeDrafts: Boolean) { - datasetcount( - sourceType: $sourceType - sourceUrl: $sourceUrl - theme: $theme - organization: $organization - subtheme: $subtheme - includeDrafts: $includeDrafts - ) { - count - iri - } -} - `; - -export function useDatasetCountQuery(options: Omit, 'query'> = {}) { - return Urql.useQuery({ query: DatasetCountDocument, ...options }); }; \ No newline at end of file diff --git a/app/graphql/resolver-types.ts b/app/graphql/resolver-types.ts index 2a66d770c..4566a2190 100644 --- a/app/graphql/resolver-types.ts +++ b/app/graphql/resolver-types.ts @@ -120,12 +120,6 @@ export type DataCubeTheme = { label?: Maybe; }; -export type DatasetCount = { - __typename?: 'DatasetCount'; - iri: Scalars['String']; - count: Scalars['Int']; -}; - export type Dimension = { iri: Scalars['String']; label: Scalars['String']; @@ -396,7 +390,6 @@ export type Query = { themes: Array; subthemes: Array; organizations: Array; - datasetcount?: Maybe>; }; @@ -453,16 +446,6 @@ export type QueryOrganizationsArgs = { }; -export type QueryDatasetcountArgs = { - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - theme?: Maybe; - organization?: Maybe; - subtheme?: Maybe; - includeDrafts?: Maybe; -}; - - export type RelatedDimension = { __typename?: 'RelatedDimension'; type: Scalars['String']; @@ -657,7 +640,6 @@ export type ResolversTypes = ResolversObject<{ DataCubeResultOrder: DataCubeResultOrder; DataCubeSearchFilter: DataCubeSearchFilter; DataCubeTheme: ResolverTypeWrapper; - DatasetCount: ResolverTypeWrapper; Dimension: ResolverTypeWrapper; DimensionValue: ResolverTypeWrapper; FilterValue: ResolverTypeWrapper; @@ -698,7 +680,6 @@ export type ResolversParentTypes = ResolversObject<{ Float: Scalars['Float']; DataCubeSearchFilter: DataCubeSearchFilter; DataCubeTheme: DataCubeTheme; - DatasetCount: DatasetCount; Dimension: ResolvedDimension; DimensionValue: Scalars['DimensionValue']; FilterValue: Scalars['FilterValue']; @@ -770,12 +751,6 @@ export type DataCubeThemeResolvers; }>; -export type DatasetCountResolvers = ResolversObject<{ - iri?: Resolver; - count?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}>; - export type DimensionResolvers = ResolversObject<{ __resolveType: TypeResolveFn<'GeoCoordinatesDimension' | 'GeoShapesDimension' | 'NominalDimension' | 'NumericalMeasure' | 'OrdinalDimension' | 'OrdinalMeasure' | 'StandardErrorDimension' | 'TemporalDimension' | 'TemporalOrdinalDimension', ParentType, ContextType>; iri?: Resolver; @@ -961,7 +936,6 @@ export type QueryResolvers, ParentType, ContextType, RequireFields>; subthemes?: Resolver, ParentType, ContextType, RequireFields>; organizations?: Resolver, ParentType, ContextType, RequireFields>; - datasetcount?: Resolver>, ParentType, ContextType, RequireFields>; }>; export interface RawObservationScalarConfig extends GraphQLScalarTypeConfig { @@ -1037,7 +1011,6 @@ export type Resolvers = ResolversObject<{ DataCubeOrganization?: DataCubeOrganizationResolvers; DataCubeResult?: DataCubeResultResolvers; DataCubeTheme?: DataCubeThemeResolvers; - DatasetCount?: DatasetCountResolvers; Dimension?: DimensionResolvers; DimensionValue?: GraphQLScalarType; FilterValue?: GraphQLScalarType; diff --git a/app/graphql/resolvers/index.ts b/app/graphql/resolvers/index.ts index 2524ad42f..b379121e7 100644 --- a/app/graphql/resolvers/index.ts +++ b/app/graphql/resolvers/index.ts @@ -43,10 +43,6 @@ export const Query: QueryResolvers = { const source = getSource(args.sourceType); return await source.organizations(parent, args, context, info); }, - datasetcount: async (parent, args, context, info) => { - const source = getSource(args.sourceType); - return await source.datasetcount(parent, args, context, info); - }, }; const DataCube: DataCubeResolvers = { diff --git a/app/graphql/resolvers/rdf.ts b/app/graphql/resolvers/rdf.ts index 95e80be32..c90d39b8f 100644 --- a/app/graphql/resolvers/rdf.ts +++ b/app/graphql/resolvers/rdf.ts @@ -24,9 +24,6 @@ import { loadOrganizations, loadSubthemes, loadThemes, - queryDatasetCountByOrganization, - queryDatasetCountBySubTheme, - queryDatasetCountByTheme, } from "@/rdf/query-cube-metadata"; import { unversionObservation } from "@/rdf/query-dimension-values"; import { queryHierarchy } from "@/rdf/query-hierarchies"; @@ -69,7 +66,6 @@ export const dataCubes: NonNullable = async ( info ) => { const { sparqlClient, sparqlClientStream } = await setup(info); - const { candidates, meta } = await searchCubes({ locale, includeDrafts, @@ -79,11 +75,12 @@ export const dataCubes: NonNullable = async ( sparqlClientStream, }); - for (let query of meta.queries) { + for (const query of meta.queries) { queries.push(query); } sortResults(candidates, order, locale); + return candidates; }; @@ -172,32 +169,6 @@ export const organizations: NonNullable = return (await loadOrganizations({ locale, sparqlClient })).filter(truthy); }; -export const datasetcount: NonNullable = async ( - _, - { organization, theme, includeDrafts }, - { setup }, - info -) => { - const { sparqlClient } = await setup(info); - const byOrg = await queryDatasetCountByOrganization({ - theme: theme || undefined, - includeDrafts: includeDrafts ?? undefined, - sparqlClient, - }); - const byTheme = await queryDatasetCountByTheme({ - organization: organization || undefined, - includeDrafts: includeDrafts ?? undefined, - sparqlClient, - }); - const bySubTheme = await queryDatasetCountBySubTheme({ - theme: theme || undefined, - organization: organization || undefined, - includeDrafts: includeDrafts ?? undefined, - sparqlClient, - }); - return [...byOrg, ...byTheme, ...bySubTheme]; -}; - export const dataCubeDimensions: NonNullable = async ({ cube, locale }, { componentIris }, { setup }, info) => { const { sparqlClient, cache } = await setup(info); diff --git a/app/graphql/resolvers/sql.ts b/app/graphql/resolvers/sql.ts index 4a74447e9..befd810ac 100644 --- a/app/graphql/resolvers/sql.ts +++ b/app/graphql/resolvers/sql.ts @@ -175,11 +175,6 @@ export const organizations: NonNullable = return []; }; -export const datasetcount: NonNullable = - async () => { - return []; - }; - export const dataCubeDimensions: NonNullable = async ({ cube }) => { // FIXME: type of cube should be different for RDF and SQL. diff --git a/app/graphql/schema.graphql b/app/graphql/schema.graphql index 51c33d7c4..a42ee8864 100644 --- a/app/graphql/schema.graphql +++ b/app/graphql/schema.graphql @@ -351,11 +351,6 @@ enum DataCubeResultOrder { CREATED_DESC } -type DatasetCount { - iri: String! - count: Int! -} - # The "Query" type is special: it lists all of the available queries that # clients can execute, along with the return type for each. type Query { @@ -400,12 +395,4 @@ type Query { sourceUrl: String! locale: String! ): [DataCubeOrganization!]! - datasetcount( - sourceType: String! - sourceUrl: String! - theme: String - organization: String - subtheme: String - includeDrafts: Boolean - ): [DatasetCount!] } diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index ebc009aa3..9cddbc377 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -103,14 +103,14 @@ export const searchCubes = async ({ // Search cubeIris along with their score const themeValues = - filters?.filter((x) => x.type === "DataCubeTheme").map((v) => v.value) || + filters?.filter((x) => x.type === "DataCubeTheme").map((v) => v.value) ?? []; const creatorValues = filters ?.filter((x) => x.type === "DataCubeOrganization") - .map((v) => v.value) || []; + .map((v) => v.value) ?? []; const aboutValues = - filters?.filter((x) => x.type === "DataCubeAbout").map((v) => v.value) || + filters?.filter((x) => x.type === "DataCubeAbout").map((v) => v.value) ?? []; const scoresQuery = SELECT.DISTINCT`?lang ?cube ?versionHistory ?name ?description ?publisher ?themeName ?creatorLabel` From 56211938d0ad977ac9d857ca211762c0c6009065 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Wed, 25 Oct 2023 17:42:38 +0200 Subject: [PATCH 03/48] perf: Do not use two queries when searching for cubes --- app/browser/context.tsx | 14 +- app/browser/dataset-browse.tsx | 51 +++-- app/browser/select-dataset-step.tsx | 4 +- app/components/debug-search.tsx | 72 ++++--- app/docs/dataset-browse.docs.tsx | 10 + app/graphql/queries/data-cubes.graphql | 17 +- app/graphql/query-hooks.ts | 84 ++++---- app/graphql/resolver-types.ts | 106 ++++++---- app/graphql/resolvers/index.ts | 4 +- app/graphql/resolvers/rdf.ts | 29 ++- app/graphql/resolvers/sql.ts | 15 +- app/graphql/schema.graphql | 26 ++- app/pages/browse/index.tsx | 4 +- app/rdf/cube-filters.ts | 6 +- app/rdf/query-search-score-utils.ts | 56 +++--- app/rdf/query-search.spec.ts | 37 ++-- app/rdf/query-search.ts | 263 ++++++++++++++++--------- 17 files changed, 460 insertions(+), 338 deletions(-) diff --git a/app/browser/context.tsx b/app/browser/context.tsx index 7e936c399..3222beb6c 100644 --- a/app/browser/context.tsx +++ b/app/browser/context.tsx @@ -8,7 +8,7 @@ import { Router, useRouter } from "next/router"; import React, { useContext, useEffect, useMemo, useRef, useState } from "react"; import { - DataCubeResultOrder, + SearchCubeResultOrder, useOrganizationsQuery, useThemesQuery, } from "@/graphql/query-hooks"; @@ -156,13 +156,13 @@ export const useBrowseState = () => { const setIncludeDrafts = useEvent((v: boolean) => setParams({ includeDrafts: v }) ); - const setOrder = useEvent((v: DataCubeResultOrder) => + const setOrder = useEvent((v: SearchCubeResultOrder) => setParams({ order: v }) ); const setDataset = useEvent((v: string) => setParams({ dataset: v })); - const previousOrderRef = useRef( - DataCubeResultOrder.Score + const previousOrderRef = useRef( + SearchCubeResultOrder.Score ); return useMemo( @@ -171,20 +171,20 @@ export const useBrowseState = () => { includeDrafts: !!includeDrafts, setIncludeDrafts, onReset: () => { - setParams({ search: "", order: DataCubeResultOrder.CreatedDesc }); + setParams({ search: "", order: SearchCubeResultOrder.CreatedDesc }); }, onSubmitSearch: (newSearch: string) => { setParams({ search: newSearch, order: newSearch === "" - ? DataCubeResultOrder.CreatedDesc + ? SearchCubeResultOrder.CreatedDesc : previousOrderRef.current, }); }, search, order, - onSetOrder: (order: DataCubeResultOrder) => { + onSetOrder: (order: SearchCubeResultOrder) => { previousOrderRef.current = order; setOrder(order); }, diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index 567c68467..3be05703d 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -36,9 +36,9 @@ import { truthy } from "@/domain/types"; import { useFormatDate } from "@/formatters"; import { DataCubeOrganization, - DataCubeResultOrder, - DataCubesQuery, DataCubeTheme, + SearchCubeResultOrder, + SearchCubesQuery, useOrganizationsQuery, useSubthemesQuery, useThemesQuery, @@ -119,7 +119,7 @@ export const SearchDatasetControls = ({ searchResult, }: { browseState: BrowseState; - searchResult: Maybe; + searchResult: Maybe; }) => { const { inputRef, @@ -131,18 +131,18 @@ export const SearchDatasetControls = ({ onSetOrder, } = browseState; - const order = stateOrder || DataCubeResultOrder.CreatedDesc; + const order = stateOrder || SearchCubeResultOrder.CreatedDesc; const options = [ { - value: DataCubeResultOrder.Score, + value: SearchCubeResultOrder.Score, label: t({ id: "dataset.order.relevance", message: `Relevance` }), }, { - value: DataCubeResultOrder.TitleAsc, + value: SearchCubeResultOrder.TitleAsc, label: t({ id: "dataset.order.title", message: `Title` }), }, { - value: DataCubeResultOrder.CreatedDesc, + value: SearchCubeResultOrder.CreatedDesc, label: t({ id: "dataset.order.newest", message: `Newest` }), }, ]; @@ -166,10 +166,10 @@ export const SearchDatasetControls = ({ aria-live="polite" data-testid="search-results-count" > - {searchResult && searchResult.dataCubes.length > 0 && ( + {searchResult && searchResult.searchCubes.length > 0 && ( { - onSetOrder(e.target.value as DataCubeResultOrder); + onSetOrder(e.target.value as SearchCubeResultOrder); }} /> @@ -602,7 +602,7 @@ const NavSection = ({ ); }; -export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { +export const SearchFilters = ({ data }: { data?: SearchCubesQuery }) => { const { dataSource } = useDataSourceStore(); const locale = useLocale(); const { filters } = useBrowseContext(); @@ -622,22 +622,24 @@ export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { }); const counts = useMemo(() => { - if (!data?.dataCubes) { + if (!data?.searchCubes) { return {}; } const result: Record = {}; - for (const { dataCube } of data.dataCubes) { - const countables = [...dataCube.themes, dataCube.creator].filter(truthy); + for (const { cube } of data.searchCubes) { + const countables = [...cube.themes, cube.creator].filter(truthy); for (const { iri } of countables) { - result[iri] = (result[iri] ?? 0) + 1; + if (iri) { + result[iri] = (result[iri] ?? 0) + 1; + } } } return result; - }, [data?.dataCubes]); + }, [data?.searchCubes]); const themeFilter = filters.find(isAttrEqual("__typename", "DataCubeTheme")); const orgFilter = filters.find( @@ -755,11 +757,9 @@ export const SearchFilters = ({ data }: { data?: DataCubesQuery }) => { }; export const DatasetResults = ({ - resultProps, query, }: { - resultProps?: Partial; - query: UseQueryState; + query: UseQueryState; }) => { const { fetching, data, error } = query; @@ -779,7 +779,7 @@ export const DatasetResults = ({ ); } - if ((data && data.dataCubes.length === 0) || !data) { + if ((data && data.searchCubes.length === 0) || !data) { return ( - {data.dataCubes.map( - ({ dataCube, highlightedTitle, highlightedDescription }) => ( + {data.searchCubes.map( + ({ cube, highlightedTitle, highlightedDescription }) => ( @@ -838,7 +837,7 @@ export const DateFormat = ({ date }: { date: string }) => { type ResultProps = { dataCube: Pick< - DataCubesQuery["dataCubes"][0]["dataCube"], + SearchCubesQuery["searchCubes"][0]["cube"], | "iri" | "publicationStatus" | "title" diff --git a/app/browser/select-dataset-step.tsx b/app/browser/select-dataset-step.tsx index c5dfba951..58a5f07f9 100644 --- a/app/browser/select-dataset-step.tsx +++ b/app/browser/select-dataset-step.tsx @@ -31,7 +31,7 @@ import { PanelLeftWrapper, PanelMiddleWrapper, } from "@/configurator/components/layout"; -import { useDataCubesQuery } from "@/graphql/query-hooks"; +import { useSearchCubesQuery } from "@/graphql/query-hooks"; import { Icon } from "@/icons"; import { useConfiguratorState, useLocale } from "@/src"; @@ -142,7 +142,7 @@ const SelectDatasetStepContent = () => { return formatBackLink(router.query); }, [router.query]); // Use the debounced query value here only! - const [datacubesQuery] = useDataCubesQuery({ + const [datacubesQuery] = useSearchCubesQuery({ variables: { sourceType: configState.dataSource.type, sourceUrl: configState.dataSource.url, diff --git a/app/components/debug-search.tsx b/app/components/debug-search.tsx index f18678c2b..836602e25 100644 --- a/app/components/debug-search.tsx +++ b/app/components/debug-search.tsx @@ -10,14 +10,9 @@ import Select from "@mui/material/Select"; import Switch from "@mui/material/Switch"; import TextField from "@mui/material/TextField"; import Typography from "@mui/material/Typography"; -import React, { - useRef, - useState, - useEffect, - KeyboardEventHandler, -} from "react"; +import { KeyboardEventHandler, useEffect, useRef, useState } from "react"; -import { DataCubeSearchFilter, useDataCubesQuery } from "@/graphql/query-hooks"; +import { SearchCubeFilter, useSearchCubesQuery } from "@/graphql/query-hooks"; import { RequestQueryMeta } from "@/graphql/query-meta"; const territoryTheme = { @@ -56,7 +51,7 @@ const Search = ({ }: { query: string; locale: string; - filters: (DataCubeSearchFilter & { name: string })[]; + filters: (SearchCubeFilter & { name: string })[]; includeDrafts: boolean; sourceUrl: string; }) => { @@ -67,10 +62,10 @@ const Search = ({ startTimeRef.current = Date.now(); }, [query, locale, includeDrafts]); - const [cubes] = useDataCubesQuery({ + const [cubes] = useSearchCubesQuery({ variables: { - locale: locale, - query: query, + locale, + query, filters: filters.map(({ type, value }) => ({ type, value })), includeDrafts, sourceUrl, @@ -108,9 +103,9 @@ const Search = ({
- {cubes.data?.dataCubes.length} results |{" "} + {cubes.data?.searchCubes.length} results |{" "} - {cubes.data?.dataCubes.map((c) => { - return ( -
- - -
- {c?.dataCube?.iri} - - {c?.dataCube.themes.map((t) => ( - - ))} - - -
- ); - })} + {cubes.data?.searchCubes.map( + ({ cube, highlightedTitle, highlightedDescription }) => { + return ( +
+ + +
+ {cube.iri} + + {cube.themes.map((t) => ( + + ))} + + +
+ ); + } + )} queries diff --git a/app/docs/dataset-browse.docs.tsx b/app/docs/dataset-browse.docs.tsx index 5787f2241..a10f9f8c7 100644 --- a/app/docs/dataset-browse.docs.tsx +++ b/app/docs/dataset-browse.docs.tsx @@ -31,6 +31,11 @@ export default () => markdown` markdown` ; - highlightedTitle?: Maybe; - highlightedDescription?: Maybe; - dataCube: DataCube; -}; - -export enum DataCubeResultOrder { - Score = 'SCORE', - TitleAsc = 'TITLE_ASC', - CreatedDesc = 'CREATED_DESC' -} - -export type DataCubeSearchFilter = { - type: Scalars['String']; - value: Scalars['String']; -}; - export type DataCubeTheme = { __typename: 'DataCubeTheme'; iri: Scalars['String']; @@ -384,7 +365,7 @@ export type Query = { __typename: 'Query'; dataCubeByIri?: Maybe; possibleFilters: Array; - dataCubes: Array; + searchCubes: Array; themes: Array; subthemes: Array; organizations: Array; @@ -411,14 +392,14 @@ export type QueryPossibleFiltersArgs = { }; -export type QueryDataCubesArgs = { +export type QuerySearchCubesArgs = { sourceType: Scalars['String']; sourceUrl: Scalars['String']; locale?: Maybe; query?: Maybe; - order?: Maybe; + order?: Maybe; includeDrafts?: Maybe; - filters?: Maybe>; + filters?: Maybe>; }; @@ -457,6 +438,36 @@ export enum ScaleType { Ratio = 'Ratio' } +export type SearchCube = { + __typename: 'SearchCube'; + iri: Scalars['String']; + title: Scalars['String']; + description?: Maybe; + creator?: Maybe; + publicationStatus: DataCubePublicationStatus; + datePublished?: Maybe; + themes: Array; +}; + +export type SearchCubeFilter = { + type: Scalars['String']; + value: Scalars['String']; +}; + +export type SearchCubeResult = { + __typename: 'SearchCubeResult'; + score?: Maybe; + cube: SearchCube; + highlightedTitle?: Maybe; + highlightedDescription?: Maybe; +}; + +export enum SearchCubeResultOrder { + Score = 'SCORE', + TitleAsc = 'TITLE_ASC', + CreatedDesc = 'CREATED_DESC' +} + export type StandardErrorDimension = Dimension & { __typename: 'StandardErrorDimension'; iri: Scalars['String']; @@ -810,18 +821,18 @@ type HierarchyMetadata_TemporalOrdinalDimension_Fragment = { __typename: 'Tempor export type HierarchyMetadataFragment = HierarchyMetadata_GeoCoordinatesDimension_Fragment | HierarchyMetadata_GeoShapesDimension_Fragment | HierarchyMetadata_NominalDimension_Fragment | HierarchyMetadata_NumericalMeasure_Fragment | HierarchyMetadata_OrdinalDimension_Fragment | HierarchyMetadata_OrdinalMeasure_Fragment | HierarchyMetadata_StandardErrorDimension_Fragment | HierarchyMetadata_TemporalDimension_Fragment | HierarchyMetadata_TemporalOrdinalDimension_Fragment; -export type DataCubesQueryVariables = Exact<{ +export type SearchCubesQueryVariables = Exact<{ sourceType: Scalars['String']; sourceUrl: Scalars['String']; locale: Scalars['String']; query?: Maybe; - order?: Maybe; + order?: Maybe; includeDrafts?: Maybe; - filters?: Maybe | DataCubeSearchFilter>; + filters?: Maybe | SearchCubeFilter>; }>; -export type DataCubesQuery = { __typename: 'Query', dataCubes: Array<{ __typename: 'DataCubeResult', highlightedTitle?: Maybe, highlightedDescription?: Maybe, dataCube: { __typename: 'DataCube', iri: string, title: string, workExamples?: Maybe>>, description?: Maybe, publicationStatus: DataCubePublicationStatus, datePublished?: Maybe, creator?: Maybe<{ __typename: 'DataCubeOrganization', iri: string, label?: Maybe }>, themes: Array<{ __typename: 'DataCubeTheme', iri: string, label?: Maybe }> } }> }; +export type SearchCubesQuery = { __typename: 'Query', searchCubes: Array<{ __typename: 'SearchCubeResult', highlightedTitle?: Maybe, highlightedDescription?: Maybe, cube: { __typename: 'SearchCube', iri: string, title: string, description?: Maybe, publicationStatus: DataCubePublicationStatus, datePublished?: Maybe, creator?: Maybe<{ __typename: 'DataCubeOrganization', iri: string, label?: Maybe }>, themes: Array<{ __typename: 'DataCubeTheme', iri: string, label?: Maybe }> } }> }; export type DataCubePreviewQueryVariables = Exact<{ iri: Scalars['String']; @@ -1225,9 +1236,9 @@ export const DimensionMetadataWithHierarchiesFragmentDoc = gql` ...hierarchyMetadata } ${HierarchyMetadataFragmentDoc}`; -export const DataCubesDocument = gql` - query DataCubes($sourceType: String!, $sourceUrl: String!, $locale: String!, $query: String, $order: DataCubeResultOrder, $includeDrafts: Boolean, $filters: [DataCubeSearchFilter!]) { - dataCubes( +export const SearchCubesDocument = gql` + query SearchCubes($sourceType: String!, $sourceUrl: String!, $locale: String!, $query: String, $order: SearchCubeResultOrder, $includeDrafts: Boolean, $filters: [SearchCubeFilter!]) { + searchCubes( sourceType: $sourceType sourceUrl: $sourceUrl locale: $locale @@ -1238,17 +1249,16 @@ export const DataCubesDocument = gql` ) { highlightedTitle highlightedDescription - dataCube { + cube { iri title - workExamples + description + publicationStatus + datePublished creator { iri label } - description - publicationStatus - datePublished themes { iri label @@ -1258,8 +1268,8 @@ export const DataCubesDocument = gql` } `; -export function useDataCubesQuery(options: Omit, 'query'> = {}) { - return Urql.useQuery({ query: DataCubesDocument, ...options }); +export function useSearchCubesQuery(options: Omit, 'query'> = {}) { + return Urql.useQuery({ query: SearchCubesDocument, ...options }); }; export const DataCubePreviewDocument = gql` query DataCubePreview($iri: String!, $sourceType: String!, $sourceUrl: String!, $locale: String!, $latest: Boolean, $filters: Filters, $disableValuesLoad: Boolean = true) { diff --git a/app/graphql/resolver-types.ts b/app/graphql/resolver-types.ts index 4566a2190..272208a04 100644 --- a/app/graphql/resolver-types.ts +++ b/app/graphql/resolver-types.ts @@ -9,7 +9,6 @@ export type Maybe = T | null; export type Exact = { [K in keyof T]: T[K] }; export type MakeOptional = Omit & { [SubKey in K]?: Maybe }; export type MakeMaybe = Omit & { [SubKey in K]: Maybe }; -export type Omit = Pick>; export type RequireFields = { [X in Exclude]?: T[X] } & { [P in K]-?: NonNullable }; /** All built-in and custom scalars, mapped to their actual values */ export type Scalars = { @@ -95,25 +94,6 @@ export enum DataCubePublicationStatus { Published = 'PUBLISHED' } -export type DataCubeResult = { - __typename?: 'DataCubeResult'; - score?: Maybe; - highlightedTitle?: Maybe; - highlightedDescription?: Maybe; - dataCube: DataCube; -}; - -export enum DataCubeResultOrder { - Score = 'SCORE', - TitleAsc = 'TITLE_ASC', - CreatedDesc = 'CREATED_DESC' -} - -export type DataCubeSearchFilter = { - type: Scalars['String']; - value: Scalars['String']; -}; - export type DataCubeTheme = { __typename?: 'DataCubeTheme'; iri: Scalars['String']; @@ -386,7 +366,7 @@ export type Query = { __typename?: 'Query'; dataCubeByIri?: Maybe; possibleFilters: Array; - dataCubes: Array; + searchCubes: Array; themes: Array; subthemes: Array; organizations: Array; @@ -413,14 +393,14 @@ export type QueryPossibleFiltersArgs = { }; -export type QueryDataCubesArgs = { +export type QuerySearchCubesArgs = { sourceType: Scalars['String']; sourceUrl: Scalars['String']; locale?: Maybe; query?: Maybe; - order?: Maybe; + order?: Maybe; includeDrafts?: Maybe; - filters?: Maybe>; + filters?: Maybe>; }; @@ -459,6 +439,36 @@ export enum ScaleType { Ratio = 'Ratio' } +export type SearchCube = { + __typename?: 'SearchCube'; + iri: Scalars['String']; + title: Scalars['String']; + description?: Maybe; + creator?: Maybe; + publicationStatus: DataCubePublicationStatus; + datePublished?: Maybe; + themes: Array; +}; + +export type SearchCubeFilter = { + type: Scalars['String']; + value: Scalars['String']; +}; + +export type SearchCubeResult = { + __typename?: 'SearchCubeResult'; + score?: Maybe; + cube: SearchCube; + highlightedTitle?: Maybe; + highlightedDescription?: Maybe; +}; + +export enum SearchCubeResultOrder { + Score = 'SCORE', + TitleAsc = 'TITLE_ASC', + CreatedDesc = 'CREATED_DESC' +} + export type StandardErrorDimension = Dimension & { __typename?: 'StandardErrorDimension'; iri: Scalars['String']; @@ -635,16 +645,13 @@ export type ResolversTypes = ResolversObject<{ Boolean: ResolverTypeWrapper; DataCubeOrganization: ResolverTypeWrapper; DataCubePublicationStatus: DataCubePublicationStatus; - DataCubeResult: ResolverTypeWrapper & { dataCube: ResolversTypes['DataCube'] }>; - Float: ResolverTypeWrapper; - DataCubeResultOrder: DataCubeResultOrder; - DataCubeSearchFilter: DataCubeSearchFilter; DataCubeTheme: ResolverTypeWrapper; Dimension: ResolverTypeWrapper; DimensionValue: ResolverTypeWrapper; FilterValue: ResolverTypeWrapper; Filters: ResolverTypeWrapper; GeoCoordinates: ResolverTypeWrapper; + Float: ResolverTypeWrapper; GeoCoordinatesDimension: ResolverTypeWrapper; GeoShapes: ResolverTypeWrapper; GeoShapesDimension: ResolverTypeWrapper; @@ -661,6 +668,10 @@ export type ResolversTypes = ResolversObject<{ RawObservation: ResolverTypeWrapper; RelatedDimension: ResolverTypeWrapper; ScaleType: ScaleType; + SearchCube: ResolverTypeWrapper; + SearchCubeFilter: SearchCubeFilter; + SearchCubeResult: ResolverTypeWrapper; + SearchCubeResultOrder: SearchCubeResultOrder; StandardErrorDimension: ResolverTypeWrapper; TemporalDimension: ResolverTypeWrapper; TemporalOrdinalDimension: ResolverTypeWrapper; @@ -676,15 +687,13 @@ export type ResolversParentTypes = ResolversObject<{ Int: Scalars['Int']; Boolean: Scalars['Boolean']; DataCubeOrganization: DataCubeOrganization; - DataCubeResult: Omit & { dataCube: ResolversParentTypes['DataCube'] }; - Float: Scalars['Float']; - DataCubeSearchFilter: DataCubeSearchFilter; DataCubeTheme: DataCubeTheme; Dimension: ResolvedDimension; DimensionValue: Scalars['DimensionValue']; FilterValue: Scalars['FilterValue']; Filters: Scalars['Filters']; GeoCoordinates: GeoCoordinates; + Float: Scalars['Float']; GeoCoordinatesDimension: ResolvedDimension; GeoShapes: Scalars['GeoShapes']; GeoShapesDimension: ResolvedDimension; @@ -700,6 +709,9 @@ export type ResolversParentTypes = ResolversObject<{ Query: {}; RawObservation: Scalars['RawObservation']; RelatedDimension: RelatedDimension; + SearchCube: SearchCube; + SearchCubeFilter: SearchCubeFilter; + SearchCubeResult: SearchCubeResult; StandardErrorDimension: ResolvedDimension; TemporalDimension: ResolvedDimension; TemporalOrdinalDimension: ResolvedDimension; @@ -737,14 +749,6 @@ export type DataCubeOrganizationResolvers; }>; -export type DataCubeResultResolvers = ResolversObject<{ - score?: Resolver, ParentType, ContextType>; - highlightedTitle?: Resolver, ParentType, ContextType>; - highlightedDescription?: Resolver, ParentType, ContextType>; - dataCube?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}>; - export type DataCubeThemeResolvers = ResolversObject<{ iri?: Resolver; label?: Resolver, ParentType, ContextType>; @@ -932,7 +936,7 @@ export type OrdinalMeasureResolvers = ResolversObject<{ dataCubeByIri?: Resolver, ParentType, ContextType, RequireFields>; possibleFilters?: Resolver, ParentType, ContextType, RequireFields>; - dataCubes?: Resolver, ParentType, ContextType, RequireFields>; + searchCubes?: Resolver, ParentType, ContextType, RequireFields>; themes?: Resolver, ParentType, ContextType, RequireFields>; subthemes?: Resolver, ParentType, ContextType, RequireFields>; organizations?: Resolver, ParentType, ContextType, RequireFields>; @@ -948,6 +952,25 @@ export type RelatedDimensionResolvers; }>; +export type SearchCubeResolvers = ResolversObject<{ + iri?: Resolver; + title?: Resolver; + description?: Resolver, ParentType, ContextType>; + creator?: Resolver, ParentType, ContextType>; + publicationStatus?: Resolver; + datePublished?: Resolver, ParentType, ContextType>; + themes?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}>; + +export type SearchCubeResultResolvers = ResolversObject<{ + score?: Resolver, ParentType, ContextType>; + cube?: Resolver; + highlightedTitle?: Resolver, ParentType, ContextType>; + highlightedDescription?: Resolver, ParentType, ContextType>; + __isTypeOf?: IsTypeOfResolverFn; +}>; + export type StandardErrorDimensionResolvers = ResolversObject<{ iri?: Resolver; label?: Resolver; @@ -1009,7 +1032,6 @@ export interface ValuePositionScalarConfig extends GraphQLScalarTypeConfig = ResolversObject<{ DataCube?: DataCubeResolvers; DataCubeOrganization?: DataCubeOrganizationResolvers; - DataCubeResult?: DataCubeResultResolvers; DataCubeTheme?: DataCubeThemeResolvers; Dimension?: DimensionResolvers; DimensionValue?: GraphQLScalarType; @@ -1031,6 +1053,8 @@ export type Resolvers = ResolversObject<{ Query?: QueryResolvers; RawObservation?: GraphQLScalarType; RelatedDimension?: RelatedDimensionResolvers; + SearchCube?: SearchCubeResolvers; + SearchCubeResult?: SearchCubeResultResolvers; StandardErrorDimension?: StandardErrorDimensionResolvers; TemporalDimension?: TemporalDimensionResolvers; TemporalOrdinalDimension?: TemporalOrdinalDimensionResolvers; diff --git a/app/graphql/resolvers/index.ts b/app/graphql/resolvers/index.ts index b379121e7..e9ae99793 100644 --- a/app/graphql/resolvers/index.ts +++ b/app/graphql/resolvers/index.ts @@ -19,9 +19,9 @@ const getSource = (dataSourceType: string) => { }; export const Query: QueryResolvers = { - dataCubes: async (parent, args, context, info) => { + searchCubes: async (parent, args, context, info) => { const source = getSource(args.sourceType); - return await source.dataCubes(parent, args, context, info); + return await source.searchCubes(parent, args, context, info); }, dataCubeByIri: async (parent, args, context, info) => { const source = getSource(args.sourceType); diff --git a/app/graphql/resolvers/rdf.ts b/app/graphql/resolvers/rdf.ts index c90d39b8f..3eaf38a57 100644 --- a/app/graphql/resolvers/rdf.ts +++ b/app/graphql/resolvers/rdf.ts @@ -9,10 +9,10 @@ import { truthy } from "@/domain/types"; import { Loaders } from "@/graphql/context"; import { DataCubeResolvers, - DataCubeResultOrder, DimensionResolvers, QueryResolvers, Resolvers, + SearchCubeResultOrder, } from "@/graphql/resolver-types"; import { createCubeDimensionValuesLoader, @@ -27,30 +27,30 @@ import { } from "@/rdf/query-cube-metadata"; import { unversionObservation } from "@/rdf/query-dimension-values"; import { queryHierarchy } from "@/rdf/query-hierarchies"; -import { SearchResult, searchCubes } from "@/rdf/query-search"; +import { SearchResult, searchCubes as _searchCubes } from "@/rdf/query-search"; const sortResults = ( results: SearchResult[], - order: DataCubeResultOrder | undefined | null, + order: SearchCubeResultOrder | undefined | null, locale: string | undefined | null ): SearchResult[] => { - const getCube = (r: SearchResult) => r.dataCube.data; switch (order) { - case DataCubeResultOrder.TitleAsc: + case SearchCubeResultOrder.TitleAsc: results.sort((a, b) => - getCube(a).title.localeCompare(getCube(b).title, locale ?? undefined) + a.cube.title.localeCompare(b.cube.title, locale ?? undefined) ); break; - case DataCubeResultOrder.CreatedDesc: + case SearchCubeResultOrder.CreatedDesc: case undefined: case null: results.sort((a, b) => { - const ra = getCube(a).datePublished || "0"; - const rb = getCube(b).datePublished || "0"; + const ra = a.cube.datePublished ?? "0"; + const rb = b.cube.datePublished ?? "0"; + return descending(ra, rb); }); break; - case DataCubeResultOrder.Score: + case SearchCubeResultOrder.Score: break; default: const exhaustCheck = order; @@ -59,20 +59,19 @@ const sortResults = ( return results; }; -export const dataCubes: NonNullable = async ( +export const searchCubes: NonNullable = async ( _, { locale, query, order, includeDrafts, filters }, { setup, queries }, info ) => { - const { sparqlClient, sparqlClientStream } = await setup(info); - const { candidates, meta } = await searchCubes({ + const { sparqlClient } = await setup(info); + const { candidates, meta } = await _searchCubes({ locale, includeDrafts, filters, - sparqlClient, query, - sparqlClientStream, + sparqlClient, }); for (const query of meta.queries) { diff --git a/app/graphql/resolvers/sql.ts b/app/graphql/resolvers/sql.ts index befd810ac..2b9c34e1b 100644 --- a/app/graphql/resolvers/sql.ts +++ b/app/graphql/resolvers/sql.ts @@ -113,14 +113,15 @@ const parseSQLDimension = ( }; }; -export const dataCubes: NonNullable = async () => { - const result = await fetchSQL({ path: "cubes" }); - const cubes = await result.json(); +export const searchCubes: NonNullable = + async () => { + const result = await fetchSQL({ path: "cubes" }); + const cubes = await result.json(); - return cubes.map((d: SQLCube) => ({ - dataCube: parseSQLCube(d), - })); -}; + return cubes.map((d: SQLCube) => ({ + dataCube: parseSQLCube(d), + })); + }; export const dataCubeByIri: NonNullable = async (_, { iri }) => { diff --git a/app/graphql/schema.graphql b/app/graphql/schema.graphql index a42ee8864..a7b846974 100644 --- a/app/graphql/schema.graphql +++ b/app/graphql/schema.graphql @@ -323,11 +323,21 @@ type OrdinalMeasure implements Dimension { hierarchy(sourceType: String!, sourceUrl: String!): [HierarchyValue!] } -type DataCubeResult { +type SearchCube { + iri: String! + title: String! + description: String + creator: DataCubeOrganization + publicationStatus: DataCubePublicationStatus! + datePublished: String + themes: [DataCubeTheme!]! +} + +type SearchCubeResult { score: Float + cube: SearchCube! highlightedTitle: String highlightedDescription: String - dataCube: DataCube! } type DataCubeTheme { @@ -340,12 +350,12 @@ type DataCubeOrganization { label: String } -input DataCubeSearchFilter { +input SearchCubeFilter { type: String! value: String! } -enum DataCubeResultOrder { +enum SearchCubeResultOrder { SCORE TITLE_ASC CREATED_DESC @@ -370,15 +380,15 @@ type Query { sourceUrl: String! filters: Filters! ): [ObservationFilter!]! - dataCubes( + searchCubes( sourceType: String! sourceUrl: String! locale: String query: String - order: DataCubeResultOrder + order: SearchCubeResultOrder includeDrafts: Boolean - filters: [DataCubeSearchFilter!] - ): [DataCubeResult!]! + filters: [SearchCubeFilter!] + ): [SearchCubeResult!]! themes( sourceType: String! sourceUrl: String! diff --git a/app/pages/browse/index.tsx b/app/pages/browse/index.tsx index 8b8e6fb1f..a0a6e1f18 100644 --- a/app/pages/browse/index.tsx +++ b/app/pages/browse/index.tsx @@ -1,7 +1,7 @@ import { SelectDatasetStep } from "@/browser/select-dataset-step"; import { AppLayout } from "@/components/layout"; import { ConfiguratorStateProvider } from "@/configurator/configurator-state"; -import { DataCubeResultOrder } from "@/graphql/query-hooks"; +import { SearchCubeResultOrder } from "@/graphql/query-hooks"; export type BrowseParams = { type?: "theme" | "organization" | "dataset"; @@ -10,7 +10,7 @@ export type BrowseParams = { subiri?: string; topic?: string; search?: string; - order?: DataCubeResultOrder; + order?: SearchCubeResultOrder; includeDrafts?: boolean; dataset?: string; }; diff --git a/app/rdf/cube-filters.ts b/app/rdf/cube-filters.ts index 578615702..73d8f2a6c 100644 --- a/app/rdf/cube-filters.ts +++ b/app/rdf/cube-filters.ts @@ -3,7 +3,7 @@ import rdf from "rdf-ext"; import { NamedNode } from "rdf-js"; import { truthy } from "@/domain/types"; -import { DataCubeSearchFilter } from "@/graphql/resolver-types"; +import { SearchCubeFilter } from "@/graphql/resolver-types"; import * as ns from "@/rdf/namespace"; import isAttrEqual from "../utils/is-attr-equal"; @@ -19,7 +19,7 @@ const isVisualizeCubeFilter = where( export const makeInQueryFilter = ( predicate: NamedNode, - filters: DataCubeSearchFilter[] + filters: SearchCubeFilter[] ) => { return filters.length > 0 ? Cube.filter.in( @@ -34,7 +34,7 @@ export const makeCubeFilters = ({ filters, }: { includeDrafts: boolean; - filters?: DataCubeSearchFilter[]; + filters?: SearchCubeFilter[]; }) => { const themeQueryFilter = makeInQueryFilter( ns.dcat.theme, diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index c1ae97865..69cd9c960 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -1,21 +1,19 @@ +import { ParsedRawSearchCube } from "./query-search"; + export const parseFloatZeroed = (s: string) => { const n = parseFloat(s); - if (Number.isNaN(n)) { - return 0; - } else { - return n; - } + return Number.isNaN(n) ? 0 : n; }; export const weights = { - name: 5, + title: 5, description: 2, themeName: 1, publisher: 1, creatorLabel: 1, }; export const langMultiplier = 1.5; -export const exactMatchPoints = weights["name"] * 2; +export const exactMatchPoints = weights.title * 2; const isStopword = (d: string) => { return d.length < 3 && d.toLowerCase() === d; @@ -25,63 +23,67 @@ const isStopword = (d: string) => { * From a list of cube rows containing weighted fields */ export const computeScores = ( - scoresRaw: any[], + cubes: ParsedRawSearchCube[], { query, - identifierName, - lang, + locale, }: { query?: string | null; - identifierName: string; - lang?: string | null; + locale?: string | null; } ) => { - const infoPerCube = {} as Record; + const infoPerCube: Record = {}; + if (query) { - for (let scoreRow of scoresRaw) { + for (const cube of cubes) { let score = 0; - for (let [field, weight] of Object.entries(weights)) { - const val = scoreRow[field]?.toLowerCase(); - if (!val) { + for (const [field, weight] of Object.entries(weights) as [ + keyof typeof weights, + number + ][]) { + const value = cube[field]?.toLowerCase(); + + if (!value) { continue; } - for (let tok of query.split(" ").filter((d) => !isStopword(d))) { - if (val.includes(tok.toLowerCase())) { + for (const token of query.split(" ").filter((d) => !isStopword(d))) { + if (value.includes(token.toLowerCase())) { score += weight; } } // Bonus points for exact match. - if (val.includes(query.toLowerCase())) { + if (value.includes(query.toLowerCase())) { score += exactMatchPoints; } } // Cubes with properties in the current language get a bonus, // as generally we expect the user to be interested in those. - if (scoreRow["lang"] === lang) { + if (cube.lang === locale) { score *= langMultiplier; } if ( - infoPerCube[scoreRow[identifierName]] === undefined || - score > infoPerCube[scoreRow[identifierName]].score + infoPerCube[cube.iri] === undefined || + score > infoPerCube[cube.iri].score ) { - infoPerCube[scoreRow[identifierName]] = { score }; + infoPerCube[cube.iri] = { score }; } } - for (let k of Object.keys(infoPerCube)) { + for (const k of Object.keys(infoPerCube)) { if (infoPerCube[k]?.score === 0) { delete infoPerCube[k]; } } } else { - for (let scoreRow of scoresRaw) { - infoPerCube[scoreRow[identifierName]] = { score: 1 }; + for (const cube of cubes) { + infoPerCube[cube.iri] = { score: 1 }; } } + return infoPerCube; }; diff --git a/app/rdf/query-search.spec.ts b/app/rdf/query-search.spec.ts index 3e38cbcd6..370a125d6 100644 --- a/app/rdf/query-search.spec.ts +++ b/app/rdf/query-search.spec.ts @@ -1,3 +1,4 @@ +import { ParsedRawSearchCube } from "./query-search"; import { computeScores, exactMatchPoints, @@ -5,38 +6,30 @@ import { weights, } from "./query-search-score-utils"; -// jest.mock("rdf-ext", () => ({})); -// jest.mock("@rdf-esm/data-model", () => ({})); -// jest.mock("@rdf-esm/namespace", () => ({ -// default: (x) => `${x}`, -// })); -// jest.mock("@tpluscode/rdf-string", () => ({})); jest.mock("@tpluscode/sparql-builder", () => ({})); -// jest.mock("@tpluscode/rdf-ns-builders", () => ({})); describe("compute scores", () => { const scores = [ - { lang: "en", cube: "a", name: "national" }, - { lang: "en", cube: "b", name: "national", description: "economy" }, - { lang: "de", cube: "c", creatorLabel: "national" }, - { lang: "de", cube: "d", creatorLabel: "" }, - { lang: "en", cube: "e", name: "National Economy of Switzerland" }, - ]; + { lang: "en", iri: "a", title: "national" }, + { lang: "en", iri: "b", title: "national", description: "economy" }, + { lang: "de", iri: "c", creatorLabel: "national" }, + { lang: "de", iri: "d", creatorLabel: "" }, + { lang: "en", iri: "e", title: "National Economy of Switzerland" }, + ] as unknown as ParsedRawSearchCube[]; it("should compute weighted score per cube from score rows", () => { const reduced = computeScores(scores, { query: "national economy", - identifierName: "cube", - lang: "en", + locale: "en", }); - expect(reduced["a"].score).toEqual(weights.name * langMultiplier); - expect(reduced["b"].score).toEqual( - (weights.name + weights.description) * langMultiplier + expect(reduced.a.score).toEqual(weights.title * langMultiplier); + expect(reduced.b.score).toEqual( + (weights.title + weights.description) * langMultiplier ); - expect(reduced["c"].score).toEqual(weights.creatorLabel); - expect(reduced["d"]).toBeUndefined(); - expect(reduced["e"].score).toEqual( - (weights.name * 2 + exactMatchPoints) * langMultiplier + expect(reduced.c.score).toEqual(weights.creatorLabel); + expect(reduced.d).toBeUndefined(); + expect(reduced.e.score).toEqual( + (weights.title * 2 + exactMatchPoints) * langMultiplier ); }); }); diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 9cddbc377..b6e766b83 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -1,20 +1,19 @@ import { TemplateResult } from "@tpluscode/rdf-string/lib/TemplateResult"; import { DESCRIBE, SELECT, sparql } from "@tpluscode/sparql-builder"; -import clownface from "clownface"; -import { descending } from "d3"; -import { Cube } from "rdf-cube-view-query"; -import rdf from "rdf-ext"; -import { Quad, Stream } from "rdf-js"; +import { descending, group, rollup } from "d3"; +import { Literal, NamedNode } from "rdf-js"; import StreamClient from "sparql-http-client"; import ParsingClient from "sparql-http-client/ParsingClient"; import { Awaited, truthy } from "@/domain/types"; import { RequestQueryMeta } from "@/graphql/query-meta"; -import { DataCubeSearchFilter } from "@/graphql/resolver-types"; -import { ResolvedDataCube } from "@/graphql/shared-types"; +import { + DataCubePublicationStatus, + SearchCubeFilter, +} from "@/graphql/resolver-types"; import * as ns from "@/rdf/namespace"; -import { parseCube, parseIri } from "@/rdf/parse"; import { fromStream } from "@/rdf/sparql-client"; +import { locales } from "@/src"; import { pragmas } from "./create-source"; import { computeScores, highlight } from "./query-search-score-utils"; @@ -68,20 +67,64 @@ const icontains = (left: string, right: string) => { return `CONTAINS(LCASE(${left}), LCASE("${right}"))`; }; -type ResultRow = Record; -const parseResultRow = (row: ResultRow) => - Object.fromEntries(Object.entries(row).map(([k, v]) => [k, v.value])); +// Keep in sync with the query. +type RawSearchCube = { + iri: NamedNode; + title: Literal; + description: Literal; + versionHistory: NamedNode; + publicationStatus: NamedNode; + datePublished: Literal; + creator: NamedNode; + creatorLabel: Literal; + publisher: NamedNode; + theme: NamedNode; + themeName: Literal; + lang: Literal; +}; + +export type ParsedRawSearchCube = { + [k in keyof RawSearchCube]: string; +}; + +const parseRawSearchCube = (cube: RawSearchCube): ParsedRawSearchCube => { + return { + iri: cube.iri.value, + title: cube.title.value, + description: cube.description?.value, + versionHistory: cube.versionHistory?.value, + publicationStatus: + cube.publicationStatus.value === + ns.adminVocabulary("CreativeWorkStatus/Published").value + ? DataCubePublicationStatus.Published + : DataCubePublicationStatus.Draft, + datePublished: cube.datePublished?.value, + creator: cube.creator?.value, + creatorLabel: cube.creatorLabel?.value, + publisher: cube.publisher?.value, + theme: cube.theme?.value, + themeName: cube.themeName?.value, + lang: cube.lang.value, + }; +}; const identity = (str: TemplateResult) => str; const optional = (str: TemplateResult) => sparql`OPTIONAL { ${str} }`; -const extractCubesFromStream = async (cubeStream: Stream) => { - const cubeDataset = await fromStream(rdf.dataset(), cubeStream); - const cf = clownface({ dataset: cubeDataset }); - return cf.has( - cf.namedNode("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), - ns.cube.Cube - ); +const getCubesByLocale = ( + rawGroupedCubesByLocale: Map, + locale: string +) => { + const rest = locales.filter((d) => d !== locale); + const orderedLocales = [locale, ...rest]; + + for (const orderedLocale of orderedLocales) { + const cubes = rawGroupedCubesByLocale.get(orderedLocale); + + if (cubes) { + return cubes; + } + } }; export const searchCubes = async ({ @@ -90,14 +133,12 @@ export const searchCubes = async ({ filters, includeDrafts, sparqlClient, - sparqlClientStream, }: { query?: string | null; locale?: string | null; - filters?: DataCubeSearchFilter[] | null; + filters?: SearchCubeFilter[] | null; includeDrafts?: Boolean | null; sparqlClient: ParsingClient; - sparqlClientStream: StreamClient; }) => { const queries = [] as RequestQueryMeta[]; @@ -113,42 +154,40 @@ export const searchCubes = async ({ filters?.filter((x) => x.type === "DataCubeAbout").map((v) => v.value) ?? []; - const scoresQuery = SELECT.DISTINCT`?lang ?cube ?versionHistory ?name ?description ?publisher ?themeName ?creatorLabel` + const scoresQuery = SELECT.DISTINCT`?lang ?iri ?title ?publicationStatus ?datePublished ?versionHistory ?description ?publisher ?theme ?themeName ?creator ?creatorLabel` .WHERE` - ?cube a ${ns.cube.Cube}. - ?cube ${ns.schema.name} ?name. + ?iri a ${ns.cube.Cube} . + ?iri ${ns.schema.name} ?title . - BIND(LANG(?name) as ?lang) + BIND(LANG(?title) as ?lang) - OPTIONAL { - ?cube ${ns.schema.description} ?description. - } + OPTIONAL { ?iri ${ns.schema.description} ?description . } - OPTIONAL { - ?cube ${ns.schema.about} ?about. - } + OPTIONAL { ?iri ${ns.schema.about} ?about .} - OPTIONAL { - ?versionHistory ${ns.schema.hasPart} ?cube. - } + OPTIONAL { ?versionHistory ${ns.schema.hasPart} ?iri . } - OPTIONAL { ?cube ${ns.dcterms.publisher} ?publisher. } + OPTIONAL { ?iri ${ns.dcterms.publisher} ?publisher . } + + OPTIONAL { ?iri ${ns.schema.creativeWorkStatus} ?publicationStatus . } + + OPTIONAL { ?iri ${ns.schema.datePublished} ?datePublished . } ${(themeValues.length > 0 ? identity : optional)(sparql` - ?cube ${ns.dcat.theme} ?theme. + ?iri ${ns.dcat.theme} ?theme. ?theme ${ns.schema.name} ?themeName. - `)} + `)} ${(creatorValues.length > 0 ? identity : optional)( sparql` - ?cube ${ns.dcterms.creator} ?creator. + ?iri ${ns.dcterms.creator} ?creator. ?creator ${ns.schema.name} ?creatorLabel. ` )} ${makeVisualizeDatasetFilter({ includeDrafts: !!includeDrafts, - cubeIriVar: "?cube", + cubeIriVar: "?iri", })} ${makeInFilter("about", aboutValues)} @@ -166,7 +205,8 @@ export const searchCubes = async ({ ?.split(" ") .slice(0, 1) .map( - (x) => `${icontains("?name", x)} || ${icontains("?description", x)}` + (x) => + `${icontains("?title", x)} || ${icontains("?description", x)}` ) .join(" || ")} @@ -183,9 +223,8 @@ export const searchCubes = async ({ || (bound(?creatorLabel) && ${query .split(" ") .map((x) => icontains("?creatorLabel", x)) - .join(" || ")}) - - )` + .join(" || ")}) + )` : "" } `.prologue`${pragmas}`; @@ -196,78 +235,118 @@ export const searchCubes = async ({ label: "scores1", }); - const data = scoreResults.data.map((x) => parseResultRow(x as ResultRow)); + const rawCubes = (scoreResults.data as RawSearchCube[]).map( + parseRawSearchCube + ); + const rawCubesByIriAndLang = rollup( + rawCubes, + (v) => group(v, (d) => d.lang), + (d) => d.iri + ); const versionHistoryPerCube = Object.fromEntries( - data.map((d) => [d.cube, d.versionHistory]) + rawCubes.map((d) => [d.iri, d.versionHistory]) ); - const infoPerCube = computeScores(data, { + const infoByCube = computeScores(rawCubes, { query, - identifierName: "cube", - lang: locale, + locale, }); - // Find information on cubes - // Potential optimisation: filter out cubes that are below some threshold - // under the maximum score and only retrieve those cubes - // The query could also dedup directly the version of the cubes - const cubeIris = Object.keys(infoPerCube); - - const sortedCubeIris = cubeIris.sort((a, b) => - descending(infoPerCube[a].score, infoPerCube[b].score) - ); - - const cubesQuery = DESCRIBE`${sortedCubeIris.map((x) => `<${x}>`).join(" ")}`; - if (!locale) { throw new Error("Must pass locale"); } - const { data: cubeStream, meta: cubesMeta } = - sortedCubeIris.length > 0 - ? await executeAndMeasure(sparqlClientStream, cubesQuery) - : { data: undefined, meta: undefined }; + const seen = new Set(); + const cubes = rawCubes + .map((cube) => { + const versionHistory = versionHistoryPerCube[cube.iri]; + const dedupIdentifier = versionHistory ?? cube.iri; - if (cubesMeta) { - queries.push({ - ...cubesMeta, - label: "cubes", - }); - } - const cubeNodes = cubeStream ? await extractCubesFromStream(cubeStream) : []; - const seen = new Set(); - const cubes = cubeNodes - .map((cubeNode) => { - const cube = cubeNode as unknown as Cube; - const iri = parseIri(cube); - const versionHistory = versionHistoryPerCube[iri]; - const dedupIdentifier = versionHistory || iri; if (seen.has(dedupIdentifier)) { return null; } + seen.add(dedupIdentifier); - return parseCube({ cube: cube, locale }); + + const rawCubeByLang = rawCubesByIriAndLang.get(cube.iri); + + if (!rawCubeByLang) { + return null; + } + + const localizedCubes = getCubesByLocale(rawCubeByLang, locale); + + if (!localizedCubes) { + return null; + } + + const parsedCube: any = { + iri: null, + title: null, + description: null, + creator: null, + publicationStatus: null, + datePublished: null, + themes: [], + }; + + for (const cube of localizedCubes) { + if (!parsedCube.iri) { + parsedCube.iri = cube.iri; + } + + if (!parsedCube.title) { + parsedCube.title = cube.title; + } + + if (!parsedCube.description) { + parsedCube.description = cube.description; + } + + if (!parsedCube.creator && cube.creator) { + parsedCube.creator = { + iri: cube.creator, + label: cube.creatorLabel, + }; + } + + if (!parsedCube.datePublished) { + parsedCube.datePublished = cube.datePublished; + } + + if (!parsedCube.publisher) { + parsedCube.publisher = cube.publisher; + } + + if (!parsedCube.publicationStatus) { + parsedCube.publicationStatus = cube.publicationStatus; + } + + if (cube.theme || cube.themeName) { + parsedCube.themes.push({ + iri: cube.theme, + name: cube.themeName, + }); + } + } + + return parsedCube; }) .filter(truthy); - // Sort the cubes per score using previously queries scores - const results = cubes - .filter((c): c is ResolvedDataCube => !!c?.data) + const candidates = cubes .sort((a, b) => - descending( - infoPerCube[a?.data.iri!].score, - infoPerCube[b?.data.iri!].score - ) + descending(infoByCube[a.iri].score, infoByCube[b.iri].score) ) - .map((c) => ({ - dataCube: c, - highlightedTitle: query ? highlight(c.data.title, query) : c.data.title, + .map((cube) => ({ + cube, + highlightedTitle: query ? highlight(cube.title, query) : cube.title, highlightedDescription: query - ? highlight(c.data.description, query) - : c.data.description, + ? highlight(cube.description, query) + : cube.description, })); return { - candidates: results, + candidates, meta: { queries, }, From 4ed5fdf142111baf310aec6ced80dbeeac4cc74a Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Wed, 25 Oct 2023 17:43:03 +0200 Subject: [PATCH 04/48] fix: Point to a correct graph to retrieve creator labels --- app/rdf/query-search.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index b6e766b83..2763c4932 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -5,7 +5,7 @@ import { Literal, NamedNode } from "rdf-js"; import StreamClient from "sparql-http-client"; import ParsingClient from "sparql-http-client/ParsingClient"; -import { Awaited, truthy } from "@/domain/types"; +import { truthy } from "@/domain/types"; import { RequestQueryMeta } from "@/graphql/query-meta"; import { DataCubePublicationStatus, @@ -178,13 +178,6 @@ export const searchCubes = async ({ ?theme ${ns.schema.name} ?themeName. `)} - ${(creatorValues.length > 0 ? identity : optional)( - sparql` - ?iri ${ns.dcterms.creator} ?creator. - ?creator ${ns.schema.name} ?creatorLabel. - ` - )} - ${makeVisualizeDatasetFilter({ includeDrafts: !!includeDrafts, cubeIriVar: "?iri", @@ -195,9 +188,20 @@ export const searchCubes = async ({ ${makeInFilter("creator", creatorValues)} FILTER(!BOUND(?description) || ?lang = LANG(?description)) - FILTER(!BOUND(?creatorLabel) || ?lang = LANG(?creatorLabel)) FILTER(!BOUND(?themeName) || ?lang = LANG(?themeName)) + ${(creatorValues.length > 0 ? identity : optional)(sparql` + ?iri ${ns.dcterms.creator} ?creator . + GRAPH { + ?creator a ${ns.schema.Organization} ; + ${ns.schema.inDefinedTermSet} . + OPTIONAL { + ?creator ${ns.schema.name} ?creatorLabel . + FILTER(!BOUND(?creatorLabel) || LANG(?creatorLabel) = ?lang) + } + } + `)} + ${ query ? `FILTER( From 3d23a440a48ad1ab5aeac19caeafa631dc9d5963 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Wed, 25 Oct 2023 18:17:10 +0200 Subject: [PATCH 05/48] perf: Use scalar for nested objects (GQL SearchCubes) --- app/domain/data.ts | 11 ++++++++++ app/graphql/queries/data-cubes.graphql | 10 ++------- app/graphql/query-hooks.ts | 22 ++++++++++---------- app/graphql/resolver-types.ts | 28 ++++++++++++++++++++++---- app/graphql/resolvers/index.ts | 2 +- app/graphql/schema.graphql | 7 +++++-- app/rdf/query-search.ts | 9 ++++++--- codegen.yml | 4 ++++ 8 files changed, 64 insertions(+), 29 deletions(-) diff --git a/app/domain/data.ts b/app/domain/data.ts index 5ca577f4b..2f422573b 100644 --- a/app/domain/data.ts +++ b/app/domain/data.ts @@ -70,6 +70,17 @@ export type GeoData = { symbolLayer: SymbolLayer | undefined; }; +// Extracted for performance reasons. +export type SearchCubeCreator = { + iri: string; + label: string; +}; + +export type SearchCubeThemes = { + iri: string; + label: string; +}[]; + const xmlSchema = "http://www.w3.org/2001/XMLSchema#"; export const parseRDFLiteral = (value: Literal): T => { const v = value.value; diff --git a/app/graphql/queries/data-cubes.graphql b/app/graphql/queries/data-cubes.graphql index 7a1e4aa81..7c10ceedb 100644 --- a/app/graphql/queries/data-cubes.graphql +++ b/app/graphql/queries/data-cubes.graphql @@ -105,14 +105,8 @@ query SearchCubes( description publicationStatus datePublished - creator { - iri - label - } - themes { - iri - label - } + creator + themes } } } diff --git a/app/graphql/query-hooks.ts b/app/graphql/query-hooks.ts index 11a597b1b..259067d08 100644 --- a/app/graphql/query-hooks.ts +++ b/app/graphql/query-hooks.ts @@ -2,6 +2,8 @@ import { DimensionValue } from '../domain/data'; import { QueryFilters } from '../configurator'; import { Observation } from '../domain/data'; import { RawObservation } from '../domain/data'; +import { SearchCubeCreator } from '../domain/data'; +import { SearchCubeThemes } from '../domain/data'; import gql from 'graphql-tag'; import * as Urql from 'urql'; export type Maybe = T | null; @@ -22,6 +24,8 @@ export type Scalars = { GeoShapes: any; Observation: Observation; RawObservation: RawObservation; + SearchCubeCreator: SearchCubeCreator; + SearchCubeThemes: SearchCubeThemes; ValueIdentifier: any; ValuePosition: any; }; @@ -443,12 +447,13 @@ export type SearchCube = { iri: Scalars['String']; title: Scalars['String']; description?: Maybe; - creator?: Maybe; + creator?: Maybe; publicationStatus: DataCubePublicationStatus; datePublished?: Maybe; - themes: Array; + themes: Scalars['SearchCubeThemes']; }; + export type SearchCubeFilter = { type: Scalars['String']; value: Scalars['String']; @@ -468,6 +473,7 @@ export enum SearchCubeResultOrder { CreatedDesc = 'CREATED_DESC' } + export type StandardErrorDimension = Dimension & { __typename: 'StandardErrorDimension'; iri: Scalars['String']; @@ -832,7 +838,7 @@ export type SearchCubesQueryVariables = Exact<{ }>; -export type SearchCubesQuery = { __typename: 'Query', searchCubes: Array<{ __typename: 'SearchCubeResult', highlightedTitle?: Maybe, highlightedDescription?: Maybe, cube: { __typename: 'SearchCube', iri: string, title: string, description?: Maybe, publicationStatus: DataCubePublicationStatus, datePublished?: Maybe, creator?: Maybe<{ __typename: 'DataCubeOrganization', iri: string, label?: Maybe }>, themes: Array<{ __typename: 'DataCubeTheme', iri: string, label?: Maybe }> } }> }; +export type SearchCubesQuery = { __typename: 'Query', searchCubes: Array<{ __typename: 'SearchCubeResult', highlightedTitle?: Maybe, highlightedDescription?: Maybe, cube: { __typename: 'SearchCube', iri: string, title: string, description?: Maybe, publicationStatus: DataCubePublicationStatus, datePublished?: Maybe, creator?: Maybe, themes: SearchCubeThemes } }> }; export type DataCubePreviewQueryVariables = Exact<{ iri: Scalars['String']; @@ -1255,14 +1261,8 @@ export const SearchCubesDocument = gql` description publicationStatus datePublished - creator { - iri - label - } - themes { - iri - label - } + creator + themes } } } diff --git a/app/graphql/resolver-types.ts b/app/graphql/resolver-types.ts index 272208a04..530175970 100644 --- a/app/graphql/resolver-types.ts +++ b/app/graphql/resolver-types.ts @@ -2,6 +2,8 @@ import { DimensionValue } from '../domain/data'; import { Filters } from '../configurator'; import { Observation } from '../domain/data'; import { RawObservation } from '../domain/data'; +import { SearchCubeCreator } from '../domain/data'; +import { SearchCubeThemes } from '../domain/data'; import { GraphQLResolveInfo, GraphQLScalarType, GraphQLScalarTypeConfig } from 'graphql'; import { ResolvedDataCube, ResolvedObservationsQuery, ResolvedMeasure, ResolvedDimension } from './shared-types'; import { VisualizeGraphQLContext } from './context'; @@ -23,6 +25,8 @@ export type Scalars = { GeoShapes: any; Observation: Observation; RawObservation: RawObservation; + SearchCubeCreator: SearchCubeCreator; + SearchCubeThemes: SearchCubeThemes; ValueIdentifier: any; ValuePosition: any; }; @@ -444,12 +448,13 @@ export type SearchCube = { iri: Scalars['String']; title: Scalars['String']; description?: Maybe; - creator?: Maybe; + creator?: Maybe; publicationStatus: DataCubePublicationStatus; datePublished?: Maybe; - themes: Array; + themes: Scalars['SearchCubeThemes']; }; + export type SearchCubeFilter = { type: Scalars['String']; value: Scalars['String']; @@ -469,6 +474,7 @@ export enum SearchCubeResultOrder { CreatedDesc = 'CREATED_DESC' } + export type StandardErrorDimension = Dimension & { __typename?: 'StandardErrorDimension'; iri: Scalars['String']; @@ -669,9 +675,11 @@ export type ResolversTypes = ResolversObject<{ RelatedDimension: ResolverTypeWrapper; ScaleType: ScaleType; SearchCube: ResolverTypeWrapper; + SearchCubeCreator: ResolverTypeWrapper; SearchCubeFilter: SearchCubeFilter; SearchCubeResult: ResolverTypeWrapper; SearchCubeResultOrder: SearchCubeResultOrder; + SearchCubeThemes: ResolverTypeWrapper; StandardErrorDimension: ResolverTypeWrapper; TemporalDimension: ResolverTypeWrapper; TemporalOrdinalDimension: ResolverTypeWrapper; @@ -710,8 +718,10 @@ export type ResolversParentTypes = ResolversObject<{ RawObservation: Scalars['RawObservation']; RelatedDimension: RelatedDimension; SearchCube: SearchCube; + SearchCubeCreator: Scalars['SearchCubeCreator']; SearchCubeFilter: SearchCubeFilter; SearchCubeResult: SearchCubeResult; + SearchCubeThemes: Scalars['SearchCubeThemes']; StandardErrorDimension: ResolvedDimension; TemporalDimension: ResolvedDimension; TemporalOrdinalDimension: ResolvedDimension; @@ -956,13 +966,17 @@ export type SearchCubeResolvers; title?: Resolver; description?: Resolver, ParentType, ContextType>; - creator?: Resolver, ParentType, ContextType>; + creator?: Resolver, ParentType, ContextType>; publicationStatus?: Resolver; datePublished?: Resolver, ParentType, ContextType>; - themes?: Resolver, ParentType, ContextType>; + themes?: Resolver; __isTypeOf?: IsTypeOfResolverFn; }>; +export interface SearchCubeCreatorScalarConfig extends GraphQLScalarTypeConfig { + name: 'SearchCubeCreator'; +} + export type SearchCubeResultResolvers = ResolversObject<{ score?: Resolver, ParentType, ContextType>; cube?: Resolver; @@ -971,6 +985,10 @@ export type SearchCubeResultResolvers; }>; +export interface SearchCubeThemesScalarConfig extends GraphQLScalarTypeConfig { + name: 'SearchCubeThemes'; +} + export type StandardErrorDimensionResolvers = ResolversObject<{ iri?: Resolver; label?: Resolver; @@ -1054,7 +1072,9 @@ export type Resolvers = ResolversObject<{ RawObservation?: GraphQLScalarType; RelatedDimension?: RelatedDimensionResolvers; SearchCube?: SearchCubeResolvers; + SearchCubeCreator?: GraphQLScalarType; SearchCubeResult?: SearchCubeResultResolvers; + SearchCubeThemes?: GraphQLScalarType; StandardErrorDimension?: StandardErrorDimensionResolvers; TemporalDimension?: TemporalDimensionResolvers; TemporalOrdinalDimension?: TemporalOrdinalDimensionResolvers; diff --git a/app/graphql/resolvers/index.ts b/app/graphql/resolvers/index.ts index e9ae99793..179345e39 100644 --- a/app/graphql/resolvers/index.ts +++ b/app/graphql/resolvers/index.ts @@ -60,7 +60,7 @@ const DataCube: DataCubeResolvers = { description: ({ data: { description } }) => description ?? null, dateModified: ({ data: { dateModified } }) => dateModified ?? null, datePublished: ({ data: { datePublished } }) => datePublished ?? null, - themes: ({ data: { themes } }) => themes || [], + themes: ({ data: { themes } }) => themes ?? [], creator: ({ data: { creator } }) => creator ?? null, dimensions: async (parent, args, context, info) => { const source = getSource(args.sourceType); diff --git a/app/graphql/schema.graphql b/app/graphql/schema.graphql index a7b846974..d3e212597 100644 --- a/app/graphql/schema.graphql +++ b/app/graphql/schema.graphql @@ -323,14 +323,17 @@ type OrdinalMeasure implements Dimension { hierarchy(sourceType: String!, sourceUrl: String!): [HierarchyValue!] } +scalar SearchCubeCreator +scalar SearchCubeThemes + type SearchCube { iri: String! title: String! description: String - creator: DataCubeOrganization + creator: SearchCubeCreator publicationStatus: DataCubePublicationStatus! datePublished: String - themes: [DataCubeTheme!]! + themes: SearchCubeThemes! } type SearchCubeResult { diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 2763c4932..632d18421 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -5,6 +5,7 @@ import { Literal, NamedNode } from "rdf-js"; import StreamClient from "sparql-http-client"; import ParsingClient from "sparql-http-client/ParsingClient"; +import { SearchCubeCreator, SearchCubeThemes } from "@/domain/data"; import { truthy } from "@/domain/types"; import { RequestQueryMeta } from "@/graphql/query-meta"; import { @@ -307,10 +308,12 @@ export const searchCubes = async ({ } if (!parsedCube.creator && cube.creator) { - parsedCube.creator = { + const creator: SearchCubeCreator = { iri: cube.creator, label: cube.creatorLabel, }; + + parsedCube.creator = creator; } if (!parsedCube.datePublished) { @@ -326,9 +329,9 @@ export const searchCubes = async ({ } if (cube.theme || cube.themeName) { - parsedCube.themes.push({ + (parsedCube.themes as SearchCubeThemes).push({ iri: cube.theme, - name: cube.themeName, + label: cube.themeName, }); } } diff --git a/codegen.yml b/codegen.yml index c5691609f..bbfa44eb9 100644 --- a/codegen.yml +++ b/codegen.yml @@ -20,6 +20,8 @@ generates: RawObservation: "../domain/data#RawObservation" Filters: "../configurator#QueryFilters" GeoShape: "../domain/data#GeoShape" + SearchCubeCreator: "../domain/data#SearchCubeCreator" + SearchCubeThemes: "../domain/data#SearchCubeThemes" app/graphql/resolver-types.ts: plugins: - "typescript" @@ -35,6 +37,8 @@ generates: RawObservation: "../domain/data#RawObservation" Filters: "../configurator#Filters" GeoShape: "../domain/data#GeoShape" + SearchCubeCreator: "../domain/data#SearchCubeCreator" + SearchCubeThemes: "../domain/data#SearchCubeThemes" mappers: DataCube: "./shared-types#ResolvedDataCube" ObservationsQuery: "./shared-types#ResolvedObservationsQuery" From a6220e56f92727a87095b7235ba4c22b4e61c84d Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 09:44:48 +0200 Subject: [PATCH 06/48] style: Prevent layout shift when draft tag is added --- app/browser/dataset-browse.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index 3be05703d..c4cfa87bf 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -887,7 +887,14 @@ export const DatasetResult = ({ return ( - + {datePublished ? : null} From ea5501ee2075ac8a4537e9220aba268a3e94a2a4 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 09:45:08 +0200 Subject: [PATCH 07/48] fix: Types --- app/browser/dataset-browse.tsx | 6 +++--- app/components/dataset-metadata.tsx | 17 +++++++++-------- app/components/tag.tsx | 10 ++-------- app/docs/dataset-browse.docs.tsx | 3 --- app/docs/tags.docs.tsx | 8 ++++---- 5 files changed, 18 insertions(+), 26 deletions(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index c4cfa87bf..a08994420 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -961,12 +961,12 @@ export const DatasetResult = ({ // event, otherwise we go first to page then to page onClick={(ev) => ev.stopPropagation()} > - {t.label} + {t.label} )) : null} - {creator && creator.label ? ( + {creator?.label ? ( page then to page onClick={(ev) => ev.stopPropagation()} > - {creator.label} + {creator.label} ) : null} diff --git a/app/components/dataset-metadata.tsx b/app/components/dataset-metadata.tsx index 00ab63968..82c8604fa 100644 --- a/app/components/dataset-metadata.tsx +++ b/app/components/dataset-metadata.tsx @@ -182,13 +182,14 @@ const DatasetTags = ({ Keywords - {[cube.creator, ...cube.themes].filter(truthy).map((t) => - t.label ? ( + {[cube.creator, ...cube.themes].filter(truthy).map((t) => { + const type = + t.__typename === "DataCubeTheme" ? "theme" : "organization"; + + return t.label ? ( @@ -196,7 +197,7 @@ const DatasetTags = ({ component={MUILink} // @ts-ignore underline="none" - type={t.__typename} + type={type} title={t.label || undefined} sx={{ maxWidth: "100%", @@ -209,8 +210,8 @@ const DatasetTags = ({ {t.label} - ) : null - )} + ) : null; + })} ); diff --git a/app/components/tag.tsx b/app/components/tag.tsx index 4b1653375..cc6462211 100644 --- a/app/components/tag.tsx +++ b/app/components/tag.tsx @@ -1,12 +1,7 @@ import { BoxProps, Typography, TypographyProps, styled } from "@mui/material"; import React from "react"; -import { DataCubeOrganization, DataCubeTheme } from "@/graphql/resolver-types"; - -type TagType = - | "draft" - | DataCubeTheme["__typename"] - | DataCubeOrganization["__typename"]; +type TagType = "draft" | "theme" | "organization"; const TagTypography = styled(Typography)(({ theme }) => ({ borderRadius: (theme.shape.borderRadius as number) * 1.5, @@ -41,8 +36,7 @@ const Tag = React.forwardRef< variant="caption" {...props} sx={{ - backgroundColor: - type === "DataCubeTheme" ? "success.light" : "primary.light", + backgroundColor: type === "theme" ? "success.light" : "primary.light", ...sx, }} > diff --git a/app/docs/dataset-browse.docs.tsx b/app/docs/dataset-browse.docs.tsx index a10f9f8c7..b07b9f060 100644 --- a/app/docs/dataset-browse.docs.tsx +++ b/app/docs/dataset-browse.docs.tsx @@ -32,13 +32,11 @@ export default () => markdown` dataCube={{ iri: "http://example.com/iri", creator: { - __typename: "DataCubeOrganization", iri: "http://example.com/iri", label: "BAFU", }, themes: [ { - __typename: "DataCubeTheme", label: "Administration", iri: "http://lindas.com/adminstration", }, @@ -55,7 +53,6 @@ export default () => markdown` dataCube={{ iri: "http://example.com/iri", creator: { - __typename: "DataCubeOrganization", iri: "http://example.com/iri", label: "BAFU", }, diff --git a/app/docs/tags.docs.tsx b/app/docs/tags.docs.tsx index bf9f5e79d..b6aa0c0a0 100644 --- a/app/docs/tags.docs.tsx +++ b/app/docs/tags.docs.tsx @@ -11,10 +11,10 @@ Tags are used to show organizations and themes of a dataset. ${( - Water - Pollution - Finance - BAFU + Water + Pollution + Finance + BAFU )} From 6e39852a7a42ee9acb21ee068d6ee7038244a1e6 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 11:23:23 +0200 Subject: [PATCH 08/48] refactor: Use scalar for SearchCube --- app/browser/dataset-browse.tsx | 6 +-- app/components/form.tsx | 1 + app/domain/data.ts | 21 +++++++---- app/graphql/queries/data-cubes.graphql | 10 +---- app/graphql/query-hooks.ts | 32 +++------------- app/graphql/resolver-types.ts | 51 ++++---------------------- app/graphql/schema.graphql | 13 +------ app/rdf/query-search.ts | 33 ++++++++--------- codegen.yml | 6 +-- 9 files changed, 50 insertions(+), 123 deletions(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index a08994420..a82442da4 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -69,7 +69,7 @@ export const SearchDatasetInput = ({ const searchLabel = t({ id: "dataset.search.label", - message: `Search datasets`, + message: "Search datasets", }); const placeholderLabel = t({ @@ -95,13 +95,13 @@ export const SearchDatasetInput = ({ inputRef={inputRef} id="datasetSearch" label={searchLabel} - defaultValue={search || ""} + defaultValue={search ?? ""} InputProps={{ inputProps: { "data-testid": "datasetSearch", }, onKeyPress: handleKeyPress, - onReset: onReset, + onReset, onFocus: () => setShowDraftCheckbox(true), }} placeholder={placeholderLabel} diff --git a/app/components/form.tsx b/app/components/form.tsx index 46441b20c..95bc68f7c 100644 --- a/app/components/form.tsx +++ b/app/components/form.tsx @@ -591,6 +591,7 @@ export const SearchField = ({ }, [inputRef, onReset] ); + return ( (value: Literal): T => { const v = value.value; diff --git a/app/graphql/queries/data-cubes.graphql b/app/graphql/queries/data-cubes.graphql index 7c10ceedb..c8dadf889 100644 --- a/app/graphql/queries/data-cubes.graphql +++ b/app/graphql/queries/data-cubes.graphql @@ -99,15 +99,7 @@ query SearchCubes( ) { highlightedTitle highlightedDescription - cube { - iri - title - description - publicationStatus - datePublished - creator - themes - } + cube } } diff --git a/app/graphql/query-hooks.ts b/app/graphql/query-hooks.ts index 259067d08..b5ffcdd4a 100644 --- a/app/graphql/query-hooks.ts +++ b/app/graphql/query-hooks.ts @@ -2,8 +2,7 @@ import { DimensionValue } from '../domain/data'; import { QueryFilters } from '../configurator'; import { Observation } from '../domain/data'; import { RawObservation } from '../domain/data'; -import { SearchCubeCreator } from '../domain/data'; -import { SearchCubeThemes } from '../domain/data'; +import { SearchCube } from '../domain/data'; import gql from 'graphql-tag'; import * as Urql from 'urql'; export type Maybe = T | null; @@ -24,8 +23,7 @@ export type Scalars = { GeoShapes: any; Observation: Observation; RawObservation: RawObservation; - SearchCubeCreator: SearchCubeCreator; - SearchCubeThemes: SearchCubeThemes; + SearchCube: SearchCube; ValueIdentifier: any; ValuePosition: any; }; @@ -442,17 +440,6 @@ export enum ScaleType { Ratio = 'Ratio' } -export type SearchCube = { - __typename: 'SearchCube'; - iri: Scalars['String']; - title: Scalars['String']; - description?: Maybe; - creator?: Maybe; - publicationStatus: DataCubePublicationStatus; - datePublished?: Maybe; - themes: Scalars['SearchCubeThemes']; -}; - export type SearchCubeFilter = { type: Scalars['String']; @@ -462,7 +449,7 @@ export type SearchCubeFilter = { export type SearchCubeResult = { __typename: 'SearchCubeResult'; score?: Maybe; - cube: SearchCube; + cube: Scalars['SearchCube']; highlightedTitle?: Maybe; highlightedDescription?: Maybe; }; @@ -473,7 +460,6 @@ export enum SearchCubeResultOrder { CreatedDesc = 'CREATED_DESC' } - export type StandardErrorDimension = Dimension & { __typename: 'StandardErrorDimension'; iri: Scalars['String']; @@ -838,7 +824,7 @@ export type SearchCubesQueryVariables = Exact<{ }>; -export type SearchCubesQuery = { __typename: 'Query', searchCubes: Array<{ __typename: 'SearchCubeResult', highlightedTitle?: Maybe, highlightedDescription?: Maybe, cube: { __typename: 'SearchCube', iri: string, title: string, description?: Maybe, publicationStatus: DataCubePublicationStatus, datePublished?: Maybe, creator?: Maybe, themes: SearchCubeThemes } }> }; +export type SearchCubesQuery = { __typename: 'Query', searchCubes: Array<{ __typename: 'SearchCubeResult', highlightedTitle?: Maybe, highlightedDescription?: Maybe, cube: SearchCube }> }; export type DataCubePreviewQueryVariables = Exact<{ iri: Scalars['String']; @@ -1255,15 +1241,7 @@ export const SearchCubesDocument = gql` ) { highlightedTitle highlightedDescription - cube { - iri - title - description - publicationStatus - datePublished - creator - themes - } + cube } } `; diff --git a/app/graphql/resolver-types.ts b/app/graphql/resolver-types.ts index 530175970..4334b9d94 100644 --- a/app/graphql/resolver-types.ts +++ b/app/graphql/resolver-types.ts @@ -2,8 +2,7 @@ import { DimensionValue } from '../domain/data'; import { Filters } from '../configurator'; import { Observation } from '../domain/data'; import { RawObservation } from '../domain/data'; -import { SearchCubeCreator } from '../domain/data'; -import { SearchCubeThemes } from '../domain/data'; +import { SearchCube } from '../domain/data'; import { GraphQLResolveInfo, GraphQLScalarType, GraphQLScalarTypeConfig } from 'graphql'; import { ResolvedDataCube, ResolvedObservationsQuery, ResolvedMeasure, ResolvedDimension } from './shared-types'; import { VisualizeGraphQLContext } from './context'; @@ -25,8 +24,7 @@ export type Scalars = { GeoShapes: any; Observation: Observation; RawObservation: RawObservation; - SearchCubeCreator: SearchCubeCreator; - SearchCubeThemes: SearchCubeThemes; + SearchCube: SearchCube; ValueIdentifier: any; ValuePosition: any; }; @@ -443,17 +441,6 @@ export enum ScaleType { Ratio = 'Ratio' } -export type SearchCube = { - __typename?: 'SearchCube'; - iri: Scalars['String']; - title: Scalars['String']; - description?: Maybe; - creator?: Maybe; - publicationStatus: DataCubePublicationStatus; - datePublished?: Maybe; - themes: Scalars['SearchCubeThemes']; -}; - export type SearchCubeFilter = { type: Scalars['String']; @@ -463,7 +450,7 @@ export type SearchCubeFilter = { export type SearchCubeResult = { __typename?: 'SearchCubeResult'; score?: Maybe; - cube: SearchCube; + cube: Scalars['SearchCube']; highlightedTitle?: Maybe; highlightedDescription?: Maybe; }; @@ -474,7 +461,6 @@ export enum SearchCubeResultOrder { CreatedDesc = 'CREATED_DESC' } - export type StandardErrorDimension = Dimension & { __typename?: 'StandardErrorDimension'; iri: Scalars['String']; @@ -674,12 +660,10 @@ export type ResolversTypes = ResolversObject<{ RawObservation: ResolverTypeWrapper; RelatedDimension: ResolverTypeWrapper; ScaleType: ScaleType; - SearchCube: ResolverTypeWrapper; - SearchCubeCreator: ResolverTypeWrapper; + SearchCube: ResolverTypeWrapper; SearchCubeFilter: SearchCubeFilter; SearchCubeResult: ResolverTypeWrapper; SearchCubeResultOrder: SearchCubeResultOrder; - SearchCubeThemes: ResolverTypeWrapper; StandardErrorDimension: ResolverTypeWrapper; TemporalDimension: ResolverTypeWrapper; TemporalOrdinalDimension: ResolverTypeWrapper; @@ -717,11 +701,9 @@ export type ResolversParentTypes = ResolversObject<{ Query: {}; RawObservation: Scalars['RawObservation']; RelatedDimension: RelatedDimension; - SearchCube: SearchCube; - SearchCubeCreator: Scalars['SearchCubeCreator']; + SearchCube: Scalars['SearchCube']; SearchCubeFilter: SearchCubeFilter; SearchCubeResult: SearchCubeResult; - SearchCubeThemes: Scalars['SearchCubeThemes']; StandardErrorDimension: ResolvedDimension; TemporalDimension: ResolvedDimension; TemporalOrdinalDimension: ResolvedDimension; @@ -962,19 +944,8 @@ export type RelatedDimensionResolvers; }>; -export type SearchCubeResolvers = ResolversObject<{ - iri?: Resolver; - title?: Resolver; - description?: Resolver, ParentType, ContextType>; - creator?: Resolver, ParentType, ContextType>; - publicationStatus?: Resolver; - datePublished?: Resolver, ParentType, ContextType>; - themes?: Resolver; - __isTypeOf?: IsTypeOfResolverFn; -}>; - -export interface SearchCubeCreatorScalarConfig extends GraphQLScalarTypeConfig { - name: 'SearchCubeCreator'; +export interface SearchCubeScalarConfig extends GraphQLScalarTypeConfig { + name: 'SearchCube'; } export type SearchCubeResultResolvers = ResolversObject<{ @@ -985,10 +956,6 @@ export type SearchCubeResultResolvers; }>; -export interface SearchCubeThemesScalarConfig extends GraphQLScalarTypeConfig { - name: 'SearchCubeThemes'; -} - export type StandardErrorDimensionResolvers = ResolversObject<{ iri?: Resolver; label?: Resolver; @@ -1071,10 +1038,8 @@ export type Resolvers = ResolversObject<{ Query?: QueryResolvers; RawObservation?: GraphQLScalarType; RelatedDimension?: RelatedDimensionResolvers; - SearchCube?: SearchCubeResolvers; - SearchCubeCreator?: GraphQLScalarType; + SearchCube?: GraphQLScalarType; SearchCubeResult?: SearchCubeResultResolvers; - SearchCubeThemes?: GraphQLScalarType; StandardErrorDimension?: StandardErrorDimensionResolvers; TemporalDimension?: TemporalDimensionResolvers; TemporalOrdinalDimension?: TemporalOrdinalDimensionResolvers; diff --git a/app/graphql/schema.graphql b/app/graphql/schema.graphql index d3e212597..166815f99 100644 --- a/app/graphql/schema.graphql +++ b/app/graphql/schema.graphql @@ -323,18 +323,7 @@ type OrdinalMeasure implements Dimension { hierarchy(sourceType: String!, sourceUrl: String!): [HierarchyValue!] } -scalar SearchCubeCreator -scalar SearchCubeThemes - -type SearchCube { - iri: String! - title: String! - description: String - creator: SearchCubeCreator - publicationStatus: DataCubePublicationStatus! - datePublished: String - themes: SearchCubeThemes! -} +scalar SearchCube type SearchCubeResult { score: Float diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 632d18421..e965e6219 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -5,7 +5,7 @@ import { Literal, NamedNode } from "rdf-js"; import StreamClient from "sparql-http-client"; import ParsingClient from "sparql-http-client/ParsingClient"; -import { SearchCubeCreator, SearchCubeThemes } from "@/domain/data"; +import { SearchCube } from "@/domain/data"; import { truthy } from "@/domain/types"; import { RequestQueryMeta } from "@/graphql/query-meta"; import { @@ -280,16 +280,17 @@ export const searchCubes = async ({ const localizedCubes = getCubesByLocale(rawCubeByLang, locale); - if (!localizedCubes) { + if (!localizedCubes?.length) { return null; } - const parsedCube: any = { - iri: null, - title: null, + const parsedCube: SearchCube = { + iri: localizedCubes[0].iri, + title: localizedCubes[0].title, description: null, creator: null, - publicationStatus: null, + publicationStatus: localizedCubes[0] + .publicationStatus as DataCubePublicationStatus, datePublished: null, themes: [], }; @@ -308,28 +309,23 @@ export const searchCubes = async ({ } if (!parsedCube.creator && cube.creator) { - const creator: SearchCubeCreator = { + parsedCube.creator = { iri: cube.creator, label: cube.creatorLabel, }; - - parsedCube.creator = creator; } if (!parsedCube.datePublished) { parsedCube.datePublished = cube.datePublished; } - if (!parsedCube.publisher) { - parsedCube.publisher = cube.publisher; - } - if (!parsedCube.publicationStatus) { - parsedCube.publicationStatus = cube.publicationStatus; + parsedCube.publicationStatus = + cube.publicationStatus as DataCubePublicationStatus; } if (cube.theme || cube.themeName) { - (parsedCube.themes as SearchCubeThemes).push({ + parsedCube.themes.push({ iri: cube.theme, label: cube.themeName, }); @@ -347,9 +343,10 @@ export const searchCubes = async ({ .map((cube) => ({ cube, highlightedTitle: query ? highlight(cube.title, query) : cube.title, - highlightedDescription: query - ? highlight(cube.description, query) - : cube.description, + highlightedDescription: + query && cube.description + ? highlight(cube.description, query) + : cube.description, })); return { diff --git a/codegen.yml b/codegen.yml index bbfa44eb9..ff9b37817 100644 --- a/codegen.yml +++ b/codegen.yml @@ -20,8 +20,7 @@ generates: RawObservation: "../domain/data#RawObservation" Filters: "../configurator#QueryFilters" GeoShape: "../domain/data#GeoShape" - SearchCubeCreator: "../domain/data#SearchCubeCreator" - SearchCubeThemes: "../domain/data#SearchCubeThemes" + SearchCube: "../domain/data#SearchCube" app/graphql/resolver-types.ts: plugins: - "typescript" @@ -37,8 +36,7 @@ generates: RawObservation: "../domain/data#RawObservation" Filters: "../configurator#Filters" GeoShape: "../domain/data#GeoShape" - SearchCubeCreator: "../domain/data#SearchCubeCreator" - SearchCubeThemes: "../domain/data#SearchCubeThemes" + SearchCube: "../domain/data#SearchCube" mappers: DataCube: "./shared-types#ResolvedDataCube" ObservationsQuery: "./shared-types#ResolvedObservationsQuery" From 8a11fb28728362ab3873b147022b88751e357db8 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 11:32:26 +0200 Subject: [PATCH 09/48] fix: Do not measure timing twice --- app/gql-flamegraph/devtool.tsx | 6 +++- app/graphql/resolvers/rdf.ts | 17 ++++------ app/rdf/query-search.ts | 61 ++++------------------------------ 3 files changed, 17 insertions(+), 67 deletions(-) diff --git a/app/gql-flamegraph/devtool.tsx b/app/gql-flamegraph/devtool.tsx index dbf02b317..03e81fc2e 100644 --- a/app/gql-flamegraph/devtool.tsx +++ b/app/gql-flamegraph/devtool.tsx @@ -193,9 +193,11 @@ const AccordionOperation = ({ const all = result?.extensions?.timings ? flatten(result?.extensions?.timings).sort(byStart) : []; + if (all.length === 0) { return 0; } + return maxBy(all, (x) => x.end)?.end! - minBy(all, (x) => x.start)?.start!; }, [result?.extensions?.timings]); @@ -376,9 +378,11 @@ function GqlDebug({ controller }: { controller: GraphqlOperationsController }) { const { opsStartMap, opsEndMap, reset, results } = controller; const [expandedId, setExpandedId] = useState(); + if (typeof window === "undefined") { return null; } + return (
@@ -397,7 +401,7 @@ function GqlDebug({ controller }: { controller: GraphqlOperationsController }) { expanded={expandedId === result.operation.key} start={opsStartMap.get(result.operation.key)!} end={opsEndMap.get(result.operation.key)!} - onChange={(_e, expanded) => + onChange={(_, expanded) => setExpandedId(expanded ? result.operation.key : undefined) } /> diff --git a/app/graphql/resolvers/rdf.ts b/app/graphql/resolvers/rdf.ts index 3eaf38a57..53b3b73a2 100644 --- a/app/graphql/resolvers/rdf.ts +++ b/app/graphql/resolvers/rdf.ts @@ -53,8 +53,8 @@ const sortResults = ( case SearchCubeResultOrder.Score: break; default: - const exhaustCheck = order; - return exhaustCheck; + const _exhaustiveCheck: never = order; + return _exhaustiveCheck; } return results; }; @@ -62,25 +62,20 @@ const sortResults = ( export const searchCubes: NonNullable = async ( _, { locale, query, order, includeDrafts, filters }, - { setup, queries }, + { setup }, info ) => { const { sparqlClient } = await setup(info); - const { candidates, meta } = await _searchCubes({ + const cubes = await _searchCubes({ locale, includeDrafts, filters, query, sparqlClient, }); + sortResults(cubes, order, locale); - for (const query of meta.queries) { - queries.push(query); - } - - sortResults(candidates, order, locale); - - return candidates; + return cubes; }; export const dataCubeByIri: NonNullable = diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index e965e6219..f04812774 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -1,19 +1,16 @@ import { TemplateResult } from "@tpluscode/rdf-string/lib/TemplateResult"; -import { DESCRIBE, SELECT, sparql } from "@tpluscode/sparql-builder"; +import { SELECT, sparql } from "@tpluscode/sparql-builder"; import { descending, group, rollup } from "d3"; import { Literal, NamedNode } from "rdf-js"; -import StreamClient from "sparql-http-client"; import ParsingClient from "sparql-http-client/ParsingClient"; import { SearchCube } from "@/domain/data"; import { truthy } from "@/domain/types"; -import { RequestQueryMeta } from "@/graphql/query-meta"; import { DataCubePublicationStatus, SearchCubeFilter, } from "@/graphql/resolver-types"; import * as ns from "@/rdf/namespace"; -import { fromStream } from "@/rdf/sparql-client"; import { locales } from "@/src"; import { pragmas } from "./create-source"; @@ -34,36 +31,6 @@ const makeInFilter = (varName: string, values: string[]) => { }`; }; -// It's a bit difficult ot access the types of the various sparql libraries -const exampleSelectQuery = SELECT``; -const exampleDescribeQuery = DESCRIBE``; -type SelectQuery = typeof exampleSelectQuery; -type DescribeQuery = typeof exampleDescribeQuery; -type StreamOfQuad = Parameters[1]; - -const executeAndMeasure = async ( - client: T extends SelectQuery ? ParsingClient : StreamClient, - query: T -): Promise<{ - meta: RequestQueryMeta; - data: T extends SelectQuery ? unknown[] : StreamOfQuad; -}> => { - const startTime = Date.now(); - // @ts-ignore - const data = await query.execute(client.query, { - operation: "postUrlencoded", - }); - const endTime = Date.now(); - return { - meta: { - startTime, - endTime, - text: query.build(), - }, - data, - }; -}; - const icontains = (left: string, right: string) => { return `CONTAINS(LCASE(${left}), LCASE("${right}"))`; }; @@ -141,8 +108,6 @@ export const searchCubes = async ({ includeDrafts?: Boolean | null; sparqlClient: ParsingClient; }) => { - const queries = [] as RequestQueryMeta[]; - // Search cubeIris along with their score const themeValues = filters?.filter((x) => x.type === "DataCubeTheme").map((v) => v.value) ?? @@ -234,15 +199,10 @@ export const searchCubes = async ({ } `.prologue`${pragmas}`; - const scoreResults = await executeAndMeasure(sparqlClient, scoresQuery); - queries.push({ - ...scoreResults.meta, - label: "scores1", + const scoreResults = await scoresQuery.execute(sparqlClient.query, { + operation: "postUrlencoded", }); - - const rawCubes = (scoreResults.data as RawSearchCube[]).map( - parseRawSearchCube - ); + const rawCubes = (scoreResults as RawSearchCube[]).map(parseRawSearchCube); const rawCubesByIriAndLang = rollup( rawCubes, (v) => group(v, (d) => d.lang), @@ -336,7 +296,7 @@ export const searchCubes = async ({ }) .filter(truthy); - const candidates = cubes + return cubes .sort((a, b) => descending(infoByCube[a.iri].score, infoByCube[b.iri].score) ) @@ -348,15 +308,6 @@ export const searchCubes = async ({ ? highlight(cube.description, query) : cube.description, })); - - return { - candidates, - meta: { - queries, - }, - }; }; -export type SearchResult = Awaited< - ReturnType ->["candidates"][0]; +export type SearchResult = Awaited>[0]; From f8fbe2f1e4f0b1329005ab145a8ed5cc6906d3a3 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 13:05:53 +0200 Subject: [PATCH 10/48] perf: Optimize cube search query ...and fix an issue with wrong theme label languages. --- app/rdf/query-search.ts | 195 +++++++++++++++++++--------------------- 1 file changed, 94 insertions(+), 101 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index f04812774..2c1cee065 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -1,6 +1,5 @@ -import { TemplateResult } from "@tpluscode/rdf-string/lib/TemplateResult"; -import { SELECT, sparql } from "@tpluscode/sparql-builder"; -import { descending, group, rollup } from "d3"; +import { SELECT } from "@tpluscode/sparql-builder"; +import { descending, group } from "d3"; import { Literal, NamedNode } from "rdf-js"; import ParsingClient from "sparql-http-client/ParsingClient"; @@ -41,14 +40,13 @@ type RawSearchCube = { title: Literal; description: Literal; versionHistory: NamedNode; - publicationStatus: NamedNode; + status: NamedNode; datePublished: Literal; - creator: NamedNode; + creatorIri: NamedNode; creatorLabel: Literal; publisher: NamedNode; - theme: NamedNode; - themeName: Literal; - lang: Literal; + themeIri: NamedNode; + themeLabel: Literal; }; export type ParsedRawSearchCube = { @@ -61,43 +59,51 @@ const parseRawSearchCube = (cube: RawSearchCube): ParsedRawSearchCube => { title: cube.title.value, description: cube.description?.value, versionHistory: cube.versionHistory?.value, - publicationStatus: - cube.publicationStatus.value === + status: + cube.status.value === ns.adminVocabulary("CreativeWorkStatus/Published").value ? DataCubePublicationStatus.Published : DataCubePublicationStatus.Draft, datePublished: cube.datePublished?.value, - creator: cube.creator?.value, + creatorIri: cube.creatorIri?.value, creatorLabel: cube.creatorLabel?.value, publisher: cube.publisher?.value, - theme: cube.theme?.value, - themeName: cube.themeName?.value, - lang: cube.lang.value, + themeIri: cube.themeIri?.value, + themeLabel: cube.themeLabel?.value, }; }; -const identity = (str: TemplateResult) => str; -const optional = (str: TemplateResult) => sparql`OPTIONAL { ${str} }`; - -const getCubesByLocale = ( - rawGroupedCubesByLocale: Map, - locale: string -) => { +const getOrderedLocales = (locale: string) => { const rest = locales.filter((d) => d !== locale); - const orderedLocales = [locale, ...rest]; - - for (const orderedLocale of orderedLocales) { - const cubes = rawGroupedCubesByLocale.get(orderedLocale); + return [locale, ...rest]; +}; - if (cubes) { - return cubes; - } - } +const buildLocalizedSubQuery = ( + s: string, + p: string, + o: string, + { locale }: { locale: string } +) => { + // Include the empty locale as well. + const locales = getOrderedLocales(locale).concat(""); + + return `${locales + .map( + (locale) => `OPTIONAL { + ?${s} ${p} ?${o}_${locale} . + FILTER(LANG(?${o}_${locale}) = "${locale}") + }` + ) + .join("\n")} + BIND(COALESCE(${locales + .map((locale) => `?${o}_${locale}`) + .join(", ")}) AS ?${o}) + `; }; export const searchCubes = async ({ query, - locale, + locale: _locale, filters, includeDrafts, sparqlClient, @@ -108,6 +114,7 @@ export const searchCubes = async ({ includeDrafts?: Boolean | null; sparqlClient: ParsingClient; }) => { + const locale = _locale ?? "de"; // Search cubeIris along with their score const themeValues = filters?.filter((x) => x.type === "DataCubeTheme").map((v) => v.value) ?? @@ -120,54 +127,51 @@ export const searchCubes = async ({ filters?.filter((x) => x.type === "DataCubeAbout").map((v) => v.value) ?? []; - const scoresQuery = SELECT.DISTINCT`?lang ?iri ?title ?publicationStatus ?datePublished ?versionHistory ?description ?publisher ?theme ?themeName ?creator ?creatorLabel` + const scoresQuery = SELECT.DISTINCT`?iri ?title ?status ?datePublished ?versionHistory ?description ?publisher ?themeIri ?themeLabel ?creatorIri ?creatorLabel` .WHERE` - ?iri a ${ns.cube.Cube} . - ?iri ${ns.schema.name} ?title . - - BIND(LANG(?title) as ?lang) - - OPTIONAL { ?iri ${ns.schema.description} ?description . } - - OPTIONAL { ?iri ${ns.schema.about} ?about .} - - OPTIONAL { ?versionHistory ${ns.schema.hasPart} ?iri . } - - OPTIONAL { ?iri ${ns.dcterms.publisher} ?publisher . } - - OPTIONAL { ?iri ${ns.schema.creativeWorkStatus} ?publicationStatus . } - - OPTIONAL { ?iri ${ns.schema.datePublished} ?datePublished . } - - ${(themeValues.length > 0 ? identity : optional)(sparql` - ?iri ${ns.dcat.theme} ?theme. - ?theme ${ns.schema.name} ?themeName. - `)} - - ${makeVisualizeDatasetFilter({ - includeDrafts: !!includeDrafts, - cubeIriVar: "?iri", - })} - - ${makeInFilter("about", aboutValues)} - ${makeInFilter("theme", themeValues)} - ${makeInFilter("creator", creatorValues)} - - FILTER(!BOUND(?description) || ?lang = LANG(?description)) - FILTER(!BOUND(?themeName) || ?lang = LANG(?themeName)) - - ${(creatorValues.length > 0 ? identity : optional)(sparql` - ?iri ${ns.dcterms.creator} ?creator . - GRAPH { - ?creator a ${ns.schema.Organization} ; - ${ns.schema.inDefinedTermSet} . - OPTIONAL { - ?creator ${ns.schema.name} ?creatorLabel . - FILTER(!BOUND(?creatorLabel) || LANG(?creatorLabel) = ?lang) + ?iri a ${ns.cube.Cube} . + ${buildLocalizedSubQuery("iri", "schema:name", "title", { + locale, + })} + ${buildLocalizedSubQuery("iri", "schema:description", "description", { + locale, + })} + OPTIONAL { ?versionHistory ${ns.schema.hasPart} ?iri . } + OPTIONAL { ?iri ${ns.schema.about} ?aboutIri . } + OPTIONAL { ?iri ${ns.dcterms.publisher} ?publisher . } + OPTIONAL { ?iri ${ns.schema.creativeWorkStatus} ?status . } + OPTIONAL { + ?iri ${ns.schema.datePublished} ?datePublished . + FILTER(DATATYPE(?datePublished) = ${ns.xsd.date}) + } + OPTIONAL { + ?iri ${ns.dcat.theme} ?themeIri . + ${buildLocalizedSubQuery("themeIri", "schema:name", "themeLabel", { + locale, + })} + } + ${makeVisualizeDatasetFilter({ + includeDrafts: !!includeDrafts, + cubeIriVar: "?iri", + })} + ${makeInFilter("aboutIri", aboutValues)} + ${makeInFilter("themeIri", themeValues)} + ${makeInFilter("creatorIri", creatorValues)} + OPTIONAL { + ?iri ${ns.dcterms.creator} ?creatorIri . + GRAPH { + ?creatorIri a ${ns.schema.Organization} ; + ${ + ns.schema.inDefinedTermSet + } . + ${buildLocalizedSubQuery( + "creatorIri", + "schema:name", + "creatorLabel", + { locale } + )} } } - `)} - ${ query ? `FILTER( @@ -185,9 +189,9 @@ export const searchCubes = async ({ .map((x) => icontains("?publisher", x)) .join(" || ")}) - || (bound(?themeName) && ${query + || (bound(?themeLabel) && ${query .split(" ") - .map((x) => icontains("?themeName", x)) + .map((x) => icontains("?themeLabel", x)) .join(" || ")}) || (bound(?creatorLabel) && ${query @@ -203,11 +207,7 @@ export const searchCubes = async ({ operation: "postUrlencoded", }); const rawCubes = (scoreResults as RawSearchCube[]).map(parseRawSearchCube); - const rawCubesByIriAndLang = rollup( - rawCubes, - (v) => group(v, (d) => d.lang), - (d) => d.iri - ); + const rawCubesByIri = group(rawCubes, (d) => d.iri); const versionHistoryPerCube = Object.fromEntries( rawCubes.map((d) => [d.iri, d.versionHistory]) ); @@ -232,30 +232,23 @@ export const searchCubes = async ({ seen.add(dedupIdentifier); - const rawCubeByLang = rawCubesByIriAndLang.get(cube.iri); - - if (!rawCubeByLang) { - return null; - } - - const localizedCubes = getCubesByLocale(rawCubeByLang, locale); + const rawCubes = rawCubesByIri.get(cube.iri); - if (!localizedCubes?.length) { + if (!rawCubes?.length) { return null; } const parsedCube: SearchCube = { - iri: localizedCubes[0].iri, - title: localizedCubes[0].title, + iri: rawCubes[0].iri, + title: rawCubes[0].title, description: null, creator: null, - publicationStatus: localizedCubes[0] - .publicationStatus as DataCubePublicationStatus, + publicationStatus: rawCubes[0].status as DataCubePublicationStatus, datePublished: null, themes: [], }; - for (const cube of localizedCubes) { + for (const cube of rawCubes) { if (!parsedCube.iri) { parsedCube.iri = cube.iri; } @@ -268,9 +261,9 @@ export const searchCubes = async ({ parsedCube.description = cube.description; } - if (!parsedCube.creator && cube.creator) { + if (!parsedCube.creator && cube.creatorIri) { parsedCube.creator = { - iri: cube.creator, + iri: cube.creatorIri, label: cube.creatorLabel, }; } @@ -281,13 +274,13 @@ export const searchCubes = async ({ if (!parsedCube.publicationStatus) { parsedCube.publicationStatus = - cube.publicationStatus as DataCubePublicationStatus; + cube.status as DataCubePublicationStatus; } - if (cube.theme || cube.themeName) { + if (cube.themeIri && cube.themeLabel) { parsedCube.themes.push({ - iri: cube.theme, - label: cube.themeName, + iri: cube.themeIri, + label: cube.themeLabel, }); } } From fbba91bfa1ecafc297f3640140a54aa75f9cd577 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 13:18:44 +0200 Subject: [PATCH 11/48] fix: Order cubes by version history and by desc(iri) to select newest cube --- app/rdf/query-search.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 2c1cee065..eddf559df 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -1,3 +1,4 @@ +import RDF from "@rdfjs/data-model"; import { SELECT } from "@tpluscode/sparql-builder"; import { descending, group } from "d3"; import { Literal, NamedNode } from "rdf-js"; @@ -201,7 +202,11 @@ export const searchCubes = async ({ )` : "" } - `.prologue`${pragmas}`; + ` + .ORDER() + // Important for the latter part of the query, to return the latest cube per unversioned iri. + .BY(RDF.variable("versionHistory")) + .THEN.BY(RDF.variable("iri"), true).prologue`${pragmas}`; const scoreResults = await scoresQuery.execute(sparqlClient.query, { operation: "postUrlencoded", From 467a17a866a305e59852e46e9e3653cb61e6cfe0 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 13:31:21 +0200 Subject: [PATCH 12/48] fix: Types --- app/rdf/query-search-score-utils.ts | 16 ++-------------- app/rdf/query-search.spec.ts | 15 ++++++--------- app/rdf/query-search.ts | 15 ++++----------- 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index 69cd9c960..b4519771d 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -8,7 +8,7 @@ export const parseFloatZeroed = (s: string) => { export const weights = { title: 5, description: 2, - themeName: 1, + themeLabel: 1, publisher: 1, creatorLabel: 1, }; @@ -24,13 +24,7 @@ const isStopword = (d: string) => { */ export const computeScores = ( cubes: ParsedRawSearchCube[], - { - query, - locale, - }: { - query?: string | null; - locale?: string | null; - } + { query }: { query?: string | null } ) => { const infoPerCube: Record = {}; @@ -60,12 +54,6 @@ export const computeScores = ( } } - // Cubes with properties in the current language get a bonus, - // as generally we expect the user to be interested in those. - if (cube.lang === locale) { - score *= langMultiplier; - } - if ( infoPerCube[cube.iri] === undefined || score > infoPerCube[cube.iri].score diff --git a/app/rdf/query-search.spec.ts b/app/rdf/query-search.spec.ts index 370a125d6..377b3c206 100644 --- a/app/rdf/query-search.spec.ts +++ b/app/rdf/query-search.spec.ts @@ -10,18 +10,15 @@ jest.mock("@tpluscode/sparql-builder", () => ({})); describe("compute scores", () => { const scores = [ - { lang: "en", iri: "a", title: "national" }, - { lang: "en", iri: "b", title: "national", description: "economy" }, - { lang: "de", iri: "c", creatorLabel: "national" }, - { lang: "de", iri: "d", creatorLabel: "" }, - { lang: "en", iri: "e", title: "National Economy of Switzerland" }, + { iri: "a", title: "national" }, + { iri: "b", title: "national", description: "economy" }, + { iri: "c", creatorLabel: "national" }, + { iri: "d", creatorLabel: "" }, + { iri: "e", title: "National Economy of Switzerland" }, ] as unknown as ParsedRawSearchCube[]; it("should compute weighted score per cube from score rows", () => { - const reduced = computeScores(scores, { - query: "national economy", - locale: "en", - }); + const reduced = computeScores(scores, { query: "national economy" }); expect(reduced.a.score).toEqual(weights.title * langMultiplier); expect(reduced.b.score).toEqual( (weights.title + weights.description) * langMultiplier diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index eddf559df..d463dea33 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -216,26 +216,19 @@ export const searchCubes = async ({ const versionHistoryPerCube = Object.fromEntries( rawCubes.map((d) => [d.iri, d.versionHistory]) ); - const infoByCube = computeScores(rawCubes, { - query, - locale, - }); - - if (!locale) { - throw new Error("Must pass locale"); - } + const infoByCube = computeScores(rawCubes, { query }); - const seen = new Set(); + const seenCubes = new Set(); const cubes = rawCubes .map((cube) => { const versionHistory = versionHistoryPerCube[cube.iri]; const dedupIdentifier = versionHistory ?? cube.iri; - if (seen.has(dedupIdentifier)) { + if (seenCubes.has(dedupIdentifier)) { return null; } - seen.add(dedupIdentifier); + seenCubes.add(dedupIdentifier); const rawCubes = rawCubesByIri.get(cube.iri); From 15d0b7696db6e624a0d952ad8499026137358190 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 14:47:29 +0200 Subject: [PATCH 13/48] fix: Do not filter by subthemes on the server side --- app/browser/dataset-browse.tsx | 127 +++++++++++++++------------- app/browser/select-dataset-step.tsx | 50 +++++++++-- app/domain/data.ts | 4 + app/rdf/query-search.ts | 65 +++++++++++--- 4 files changed, 164 insertions(+), 82 deletions(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index a82442da4..cb815d800 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -1,4 +1,3 @@ -import { Maybe } from "@graphql-tools/utils/types"; import { Plural, t, Trans } from "@lingui/macro"; import { Box, @@ -15,11 +14,11 @@ import { Reorder } from "framer-motion"; import orderBy from "lodash/orderBy"; import pickBy from "lodash/pickBy"; import sortBy from "lodash/sortBy"; +import uniqBy from "lodash/uniqBy"; import Link from "next/link"; import { useRouter } from "next/router"; import { stringify } from "qs"; import React, { useMemo, useState } from "react"; -import { UseQueryState } from "urql"; import Flex, { FlexProps } from "@/components/flex"; import { Checkbox, MinimalisticSelect, SearchField } from "@/components/form"; @@ -32,6 +31,7 @@ import { } from "@/components/presence"; import Tag from "@/components/tag"; import useDisclosure from "@/components/use-disclosure"; +import { SearchCube } from "@/domain/data"; import { truthy } from "@/domain/types"; import { useFormatDate } from "@/formatters"; import { @@ -40,10 +40,12 @@ import { SearchCubeResultOrder, SearchCubesQuery, useOrganizationsQuery, - useSubthemesQuery, useThemesQuery, } from "@/graphql/query-hooks"; -import { DataCubePublicationStatus } from "@/graphql/resolver-types"; +import { + DataCubePublicationStatus, + SearchCubeResult, +} from "@/graphql/resolver-types"; import SvgIcCategories from "@/icons/components/IcCategories"; import SvgIcClose from "@/icons/components/IcClose"; import SvgIcOrganisations from "@/icons/components/IcOrganisations"; @@ -116,10 +118,10 @@ export const SearchDatasetInput = ({ export const SearchDatasetControls = ({ browseState, - searchResult, + cubes, }: { browseState: BrowseState; - searchResult: Maybe; + cubes: SearchCubeResult[]; }) => { const { inputRef, @@ -166,10 +168,10 @@ export const SearchDatasetControls = ({ aria-live="polite" data-testid="search-results-count" > - {searchResult && searchResult.searchCubes.length > 0 && ( + {cubes.length > 0 && ( level === 1 ? f.__typename !== next.__typename : true ); + if (level === 1) { newFilters.push(next); } + return ( "/browse/" + newFilters.map(encodeFilter).join("/") + `?${extraURLParams}` ); @@ -350,12 +354,14 @@ const NavItem = ({ ); const nextIndex = filters.findIndex((f) => f.iri === next.iri); const newFilters = nextIndex === 0 ? [] : filters.slice(0, 1); + return ( "/browse?" + newFilters.map(encodeFilter).join("&") + `&${extraURLParams}` ); }, [includeDrafts, search, filters, next.iri]); const classes = useStyles(); + const removeFilterButton = ( ); + const countChip = count !== undefined ? ( {count} ) : null; + return ( ; - export const Subthemes = ({ - organization, + subthemes, filters, counts, }: { - organization: DataCubeOrganization; + subthemes: SearchCube["subthemes"]; filters: BrowseFilter[]; counts: Record; }) => { - const termsetIri = organizationIriToTermsetParentIri[organization.iri]; - const { dataSource } = useDataSourceStore(); - const locale = useLocale(); - const [{ data: subthemes }] = useSubthemesQuery({ - variables: { - parentIri: termsetIri, - sourceType: dataSource.type, - sourceUrl: dataSource.url, - locale, - }, - pause: !termsetIri, - }); - const alphaSubthemes = useMemo( - () => sortBy(subthemes?.subthemes, (x) => x.label), - [subthemes] - ); + const sortedSubthemes = useMemo(() => { + return sortBy(subthemes, (d) => d.label); + }, [subthemes]); + return ( <> - {alphaSubthemes.map((x) => { - const count = counts[x.iri]; + {sortedSubthemes.map((d) => { + const count = counts[d.iri]; + if (!count) { return null; } + + const filter: BrowseFilter = { + __typename: "DataCubeAbout", + ...d, + }; + return ( - {x.label} + {d.label} ); })} @@ -602,7 +600,7 @@ const NavSection = ({ ); }; -export const SearchFilters = ({ data }: { data?: SearchCubesQuery }) => { +export const SearchFilters = ({ cubes }: { cubes: SearchCubeResult[] }) => { const { dataSource } = useDataSourceStore(); const locale = useLocale(); const { filters } = useBrowseContext(); @@ -622,14 +620,14 @@ export const SearchFilters = ({ data }: { data?: SearchCubesQuery }) => { }); const counts = useMemo(() => { - if (!data?.searchCubes) { - return {}; - } - const result: Record = {}; - for (const { cube } of data.searchCubes) { - const countables = [...cube.themes, cube.creator].filter(truthy); + for (const { cube } of cubes) { + const countables = [ + ...cube.themes, + ...cube.subthemes, + cube.creator, + ].filter(truthy); for (const { iri } of countables) { if (iri) { @@ -639,7 +637,7 @@ export const SearchFilters = ({ data }: { data?: SearchCubesQuery }) => { } return result; - }, [data?.searchCubes]); + }, [cubes]); const themeFilter = filters.find(isAttrEqual("__typename", "DataCubeTheme")); const orgFilter = filters.find( @@ -703,6 +701,13 @@ export const SearchFilters = ({ data }: { data?: SearchCubesQuery }) => { /> ) : null; + const subthemes = React.useMemo(() => { + return uniqBy( + cubes.flatMap((d) => d.cube.subthemes), + (d) => d.iri + ); + }, [cubes]); + const orgNav = displayedOrgs && displayedOrgs.length > 0 ? ( { extra={ orgFilter && filters[0] === orgFilter ? ( @@ -757,12 +762,14 @@ export const SearchFilters = ({ data }: { data?: SearchCubesQuery }) => { }; export const DatasetResults = ({ - query, + fetching, + error, + cubes, }: { - query: UseQueryState; + fetching: boolean; + error: any; + cubes: SearchCubeResult[]; }) => { - const { fetching, data, error } = query; - if (fetching) { return ( @@ -779,7 +786,7 @@ export const DatasetResults = ({ ); } - if ((data && data.searchCubes.length === 0) || !data) { + if (cubes.length === 0) { return ( - {data.searchCubes.map( - ({ cube, highlightedTitle, highlightedDescription }) => ( - - ) - )} + {cubes.map(({ cube, highlightedTitle, highlightedDescription }) => ( + + ))} ); }; diff --git a/app/browser/select-dataset-step.tsx b/app/browser/select-dataset-step.tsx index 58a5f07f9..13047b60e 100644 --- a/app/browser/select-dataset-step.tsx +++ b/app/browser/select-dataset-step.tsx @@ -142,7 +142,7 @@ const SelectDatasetStepContent = () => { return formatBackLink(router.query); }, [router.query]); // Use the debounced query value here only! - const [datacubesQuery] = useSearchCubesQuery({ + const [{ data, fetching, error }] = useSearchCubesQuery({ variables: { sourceType: configState.dataSource.type, sourceUrl: configState.dataSource.url, @@ -151,9 +151,12 @@ const SelectDatasetStepContent = () => { order, includeDrafts, filters: filters - ? filters.map((filter) => { - return { type: filter.__typename, value: filter.iri }; - }) + ? filters + // Subtheme filters are used on the client side. + .filter((d) => d.__typename !== "DataCubeAbout") + .map((filter) => { + return { type: filter.__typename, value: filter.iri }; + }) : [], }, }); @@ -163,6 +166,35 @@ const SelectDatasetStepContent = () => { datasetIri: dataset, }); + const { allCubes, cubes } = React.useMemo(() => { + if (fetching || error || (data && data.searchCubes.length === 0) || !data) { + return { + allCubes: [], + cubes: [], + }; + } + + const subthemeFilters = filters.filter( + (d) => d.__typename === "DataCubeAbout" + ); + + if (subthemeFilters.length === 0) { + return { + allCubes: data.searchCubes, + cubes: data.searchCubes, + }; + } + + const subthemes = subthemeFilters.map((d) => d.iri); + + return { + allCubes: data.searchCubes, + cubes: data.searchCubes.filter((d) => { + return d.cube.subthemes.some((d) => subthemes.includes(d.iri)); + }), + }; + }, [data, error, fetching, filters]); + if (configState.state !== "SELECTING_DATASET") { return null; } @@ -237,7 +269,7 @@ const SelectDatasetStepContent = () => { ) : ( - + )} @@ -272,9 +304,13 @@ const SelectDatasetStepContent = () => { + - )} diff --git a/app/domain/data.ts b/app/domain/data.ts index 0413f30f5..acdccb7c6 100644 --- a/app/domain/data.ts +++ b/app/domain/data.ts @@ -86,6 +86,10 @@ export type SearchCube = { iri: string; label: string; }[]; + subthemes: { + iri: string; + label: string; + }[]; }; const xmlSchema = "http://www.w3.org/2001/XMLSchema#"; diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index d463dea33..93774746b 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -20,13 +20,14 @@ import { makeVisualizeDatasetFilter } from "./query-utils"; const toNamedNode = (x: string) => { return `<${x}>`; }; + const makeInFilter = (varName: string, values: string[]) => { return ` ${ values.length > 0 - ? `FILTER (bound(?${varName}) && - ?${varName} IN (${values.map(toNamedNode)}) - )` + ? `FILTER (bound(?${varName}) && ?${varName} IN (${values.map( + toNamedNode + )}))` : "" }`; }; @@ -48,6 +49,8 @@ type RawSearchCube = { publisher: NamedNode; themeIri: NamedNode; themeLabel: Literal; + subthemeIri: NamedNode; + subthemeLabel: Literal; }; export type ParsedRawSearchCube = { @@ -71,6 +74,8 @@ const parseRawSearchCube = (cube: RawSearchCube): ParsedRawSearchCube => { publisher: cube.publisher?.value, themeIri: cube.themeIri?.value, themeLabel: cube.themeLabel?.value, + subthemeIri: cube.subthemeIri?.value, + subthemeLabel: cube.subthemeLabel?.value, }; }; @@ -124,12 +129,10 @@ export const searchCubes = async ({ filters ?.filter((x) => x.type === "DataCubeOrganization") .map((v) => v.value) ?? []; - const aboutValues = - filters?.filter((x) => x.type === "DataCubeAbout").map((v) => v.value) ?? - []; - const scoresQuery = SELECT.DISTINCT`?iri ?title ?status ?datePublished ?versionHistory ?description ?publisher ?themeIri ?themeLabel ?creatorIri ?creatorLabel` - .WHERE` + const scoresQuery = SELECT.DISTINCT` + ?iri ?title ?status ?datePublished ?versionHistory ?description ?publisher ?creatorIri ?creatorLabel ?themeIri ?themeLabel ?subthemeIri ?subthemeLabel + `.WHERE` ?iri a ${ns.cube.Cube} . ${buildLocalizedSubQuery("iri", "schema:name", "title", { locale, @@ -138,7 +141,6 @@ export const searchCubes = async ({ locale, })} OPTIONAL { ?versionHistory ${ns.schema.hasPart} ?iri . } - OPTIONAL { ?iri ${ns.schema.about} ?aboutIri . } OPTIONAL { ?iri ${ns.dcterms.publisher} ?publisher . } OPTIONAL { ?iri ${ns.schema.creativeWorkStatus} ?status . } OPTIONAL { @@ -151,13 +153,26 @@ export const searchCubes = async ({ locale, })} } + OPTIONAL { + ?iri ${ns.schema.about} ?subthemeIri . + OPTIONAL { + ?subthemeIri ${ + ns.schema.inDefinedTermSet + } . + } + ${buildLocalizedSubQuery( + "subthemeIri", + "schema:name", + "subthemeLabel", + { locale } + )} + } ${makeVisualizeDatasetFilter({ includeDrafts: !!includeDrafts, cubeIriVar: "?iri", })} - ${makeInFilter("aboutIri", aboutValues)} - ${makeInFilter("themeIri", themeValues)} ${makeInFilter("creatorIri", creatorValues)} + ${makeInFilter("themeIri", themeValues)} OPTIONAL { ?iri ${ns.dcterms.creator} ?creatorIri . GRAPH { @@ -189,12 +204,17 @@ export const searchCubes = async ({ .split(" ") .map((x) => icontains("?publisher", x)) .join(" || ")}) - + || (bound(?themeLabel) && ${query .split(" ") .map((x) => icontains("?themeLabel", x)) .join(" || ")}) - + + || (bound(?subthemeLabel) && ${query + .split(" ") + .map((x) => icontains("?subthemeLabel", x)) + .join(" || ")}) + || (bound(?creatorLabel) && ${query .split(" ") .map((x) => icontains("?creatorLabel", x)) @@ -206,6 +226,7 @@ export const searchCubes = async ({ .ORDER() // Important for the latter part of the query, to return the latest cube per unversioned iri. .BY(RDF.variable("versionHistory")) + .THEN.BY(RDF.variable("datePublished"), true) .THEN.BY(RDF.variable("iri"), true).prologue`${pragmas}`; const scoreResults = await scoresQuery.execute(sparqlClient.query, { @@ -244,6 +265,7 @@ export const searchCubes = async ({ publicationStatus: rawCubes[0].status as DataCubePublicationStatus, datePublished: null, themes: [], + subthemes: [], }; for (const cube of rawCubes) { @@ -275,12 +297,27 @@ export const searchCubes = async ({ cube.status as DataCubePublicationStatus; } - if (cube.themeIri && cube.themeLabel) { + if ( + cube.themeIri && + cube.themeLabel && + !parsedCube.themes.some((d) => d.iri === cube.themeIri) + ) { parsedCube.themes.push({ iri: cube.themeIri, label: cube.themeLabel, }); } + + if ( + cube.subthemeIri && + cube.subthemeLabel && + !parsedCube.subthemes.some((d) => d.iri === cube.subthemeIri) + ) { + parsedCube.subthemes.push({ + iri: cube.subthemeIri, + label: cube.subthemeLabel, + }); + } } return parsedCube; From e4c02785206647643377e7b0a6e3acbbbac25a96 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 15:31:13 +0200 Subject: [PATCH 14/48] fix: Tests --- app/rdf/query-search-score-utils.ts | 1 - app/rdf/query-search.spec.ts | 11 +++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index b4519771d..f604ec14a 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -12,7 +12,6 @@ export const weights = { publisher: 1, creatorLabel: 1, }; -export const langMultiplier = 1.5; export const exactMatchPoints = weights.title * 2; const isStopword = (d: string) => { diff --git a/app/rdf/query-search.spec.ts b/app/rdf/query-search.spec.ts index 377b3c206..06db00f04 100644 --- a/app/rdf/query-search.spec.ts +++ b/app/rdf/query-search.spec.ts @@ -2,7 +2,6 @@ import { ParsedRawSearchCube } from "./query-search"; import { computeScores, exactMatchPoints, - langMultiplier, weights, } from "./query-search-score-utils"; @@ -19,14 +18,10 @@ describe("compute scores", () => { it("should compute weighted score per cube from score rows", () => { const reduced = computeScores(scores, { query: "national economy" }); - expect(reduced.a.score).toEqual(weights.title * langMultiplier); - expect(reduced.b.score).toEqual( - (weights.title + weights.description) * langMultiplier - ); + expect(reduced.a.score).toEqual(weights.title); + expect(reduced.b.score).toEqual(weights.title + weights.description); expect(reduced.c.score).toEqual(weights.creatorLabel); expect(reduced.d).toBeUndefined(); - expect(reduced.e.score).toEqual( - (weights.title * 2 + exactMatchPoints) * langMultiplier - ); + expect(reduced.e.score).toEqual(weights.title * 2 + exactMatchPoints); }); }); From a3b4a8fc6f485b758c3dfb25e8419485cb8d5911 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 15:39:55 +0200 Subject: [PATCH 15/48] refactor: Clean up --- app/browser/dataset-browse.tsx | 22 +++++++----------- app/browser/select-dataset-step.tsx | 36 ++++++++++++++++------------- app/rdf/cube-filters.ts | 6 ++--- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index cb815d800..f1a9bf797 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -437,28 +437,19 @@ export const Subthemes = ({ filters: BrowseFilter[]; counts: Record; }) => { - const sortedSubthemes = useMemo(() => { - return sortBy(subthemes, (d) => d.label); - }, [subthemes]); - return ( <> - {sortedSubthemes.map((d) => { + {subthemes.map((d) => { const count = counts[d.iri]; if (!count) { return null; } - const filter: BrowseFilter = { - __typename: "DataCubeAbout", - ...d, - }; - return ( { ) : null; const subthemes = React.useMemo(() => { - return uniqBy( - cubes.flatMap((d) => d.cube.subthemes), - (d) => d.iri + return sortBy( + uniqBy( + cubes.flatMap((d) => d.cube.subthemes), + (d) => d.iri + ), + (d) => d.label ); }, [cubes]); diff --git a/app/browser/select-dataset-step.tsx b/app/browser/select-dataset-step.tsx index 13047b60e..53d19aaac 100644 --- a/app/browser/select-dataset-step.tsx +++ b/app/browser/select-dataset-step.tsx @@ -42,7 +42,7 @@ import { buildURLFromBrowseState, useBrowseContext, } from "./context"; -import { DataCubeAbout } from "./filters"; +import { BrowseFilter, DataCubeAbout } from "./filters"; const softJSONParse = (v: string) => { try { @@ -126,6 +126,18 @@ export const formatBackLink = ( return buildURLFromBrowseState(backParameters); }; +const prepareSearchQueryFilters = (filters: BrowseFilter[]) => { + return ( + filters + // Subthemes are filtered on client side. + .filter( + (d): d is Exclude => + d.__typename !== "DataCubeAbout" + ) + .map((d) => ({ type: d.__typename, label: d.label, value: d.iri })) + ); +}; + const SelectDatasetStepContent = () => { const locale = useLocale(); const [configState] = useConfiguratorState(); @@ -141,6 +153,11 @@ const SelectDatasetStepContent = () => { const backLink = useMemo(() => { return formatBackLink(router.query); }, [router.query]); + + const queryFilters = useMemo(() => { + return filters ? prepareSearchQueryFilters(filters) : []; + }, [filters]); + // Use the debounced query value here only! const [{ data, fetching, error }] = useSearchCubesQuery({ variables: { @@ -150,14 +167,7 @@ const SelectDatasetStepContent = () => { query: debouncedQuery, order, includeDrafts, - filters: filters - ? filters - // Subtheme filters are used on the client side. - .filter((d) => d.__typename !== "DataCubeAbout") - .map((filter) => { - return { type: filter.__typename, value: filter.iri }; - }) - : [], + filters: queryFilters, }, }); @@ -292,13 +302,7 @@ const SelectDatasetStepContent = () => { className={classes.filters} variant="h1" > - {filters - .filter( - (f): f is Exclude => - f.__typename !== "DataCubeAbout" - ) - .map((f) => f.label) - .join(", ")} + {queryFilters.map((d) => d.label).join(", ")} )} diff --git a/app/rdf/cube-filters.ts b/app/rdf/cube-filters.ts index 73d8f2a6c..4f0bafe08 100644 --- a/app/rdf/cube-filters.ts +++ b/app/rdf/cube-filters.ts @@ -38,15 +38,15 @@ export const makeCubeFilters = ({ }) => { const themeQueryFilter = makeInQueryFilter( ns.dcat.theme, - filters?.filter(isAttrEqual("type", "DataCubeTheme")) || [] + filters?.filter(isAttrEqual("type", "DataCubeTheme")) ?? [] ); const orgQueryFilter = makeInQueryFilter( ns.dcterms.creator, - filters?.filter(isAttrEqual("type", "DataCubeOrganization")) || [] + filters?.filter(isAttrEqual("type", "DataCubeOrganization")) ?? [] ); const aboutQueryFilter = makeInQueryFilter( ns.schema.about, - filters?.filter(isAttrEqual("type", "DataCubeAbout")) || [] + filters?.filter(isAttrEqual("type", "DataCubeAbout")) ?? [] ); const res = [ From ac5e6ed3e4d26864aa8345d4fed4bd79f81ee17d Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 15:49:53 +0200 Subject: [PATCH 16/48] fix: Missing text input value after refresh --- app/browser/dataset-browse.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index f1a9bf797..4826afe6c 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -94,6 +94,7 @@ export const SearchDatasetInput = ({ return ( Date: Thu, 26 Oct 2023 16:55:33 +0200 Subject: [PATCH 17/48] fix: Type --- app/graphql/query-hooks.ts | 1 + app/graphql/resolver-types.ts | 1 + app/graphql/schema.graphql | 1 + 3 files changed, 3 insertions(+) diff --git a/app/graphql/query-hooks.ts b/app/graphql/query-hooks.ts index b5ffcdd4a..12c90b434 100644 --- a/app/graphql/query-hooks.ts +++ b/app/graphql/query-hooks.ts @@ -443,6 +443,7 @@ export enum ScaleType { export type SearchCubeFilter = { type: Scalars['String']; + label?: Maybe; value: Scalars['String']; }; diff --git a/app/graphql/resolver-types.ts b/app/graphql/resolver-types.ts index 4334b9d94..27d5afc85 100644 --- a/app/graphql/resolver-types.ts +++ b/app/graphql/resolver-types.ts @@ -444,6 +444,7 @@ export enum ScaleType { export type SearchCubeFilter = { type: Scalars['String']; + label?: Maybe; value: Scalars['String']; }; diff --git a/app/graphql/schema.graphql b/app/graphql/schema.graphql index 166815f99..04cdc66e3 100644 --- a/app/graphql/schema.graphql +++ b/app/graphql/schema.graphql @@ -344,6 +344,7 @@ type DataCubeOrganization { input SearchCubeFilter { type: String! + label: String value: String! } From 91a298c3cd2a90c7a7bff6e9c2357ec2bc892482 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 17:00:31 +0200 Subject: [PATCH 18/48] fix: Removing browse theme and organization filters --- app/browser/dataset-browse.tsx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index 4826afe6c..dbe18bfc6 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -338,9 +338,9 @@ const NavItem = ({ newFilters.push(next); } - return ( - "/browse/" + newFilters.map(encodeFilter).join("/") + `?${extraURLParams}` - ); + return `/browse/${newFilters + .map(encodeFilter) + .join("/")}?${extraURLParams}`; }, [includeDrafts, search, level, next, filters]); const removeFilterPath = useMemo(() => { @@ -353,12 +353,12 @@ const NavItem = ({ Boolean ) ); - const nextIndex = filters.findIndex((f) => f.iri === next.iri); - const newFilters = nextIndex === 0 ? [] : filters.slice(0, 1); + // const nextIndex = filters.findIndex((f) => f.iri === next.iri); + const newFilters = filters.filter((d) => d.iri !== next.iri); - return ( - "/browse?" + newFilters.map(encodeFilter).join("&") + `&${extraURLParams}` - ); + return `/browse/${newFilters + .map(encodeFilter) + .join("/")}?${extraURLParams}`; }, [includeDrafts, search, filters, next.iri]); const classes = useStyles(); From 826ebebbd942cd3179e3f95bae17d4e021382ca2 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 17:08:12 +0200 Subject: [PATCH 19/48] feat: Allow subthemes to be displayed when organization is second section --- app/browser/dataset-browse.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index dbe18bfc6..635a3f8ae 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -718,7 +718,7 @@ export const SearchFilters = ({ cubes }: { cubes: SearchCubeResult[] }) => { icon={} label={Organizations} extra={ - orgFilter && filters[0] === orgFilter ? ( + orgFilter && filters.includes(orgFilter) ? ( Date: Thu, 26 Oct 2023 17:08:30 +0200 Subject: [PATCH 20/48] fix: Selecting subthemes when both sections are filtered --- app/browser/dataset-browse.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index 635a3f8ae..610e1bd14 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -330,8 +330,10 @@ const NavItem = ({ Boolean ) ); - const newFilters = [...filters].filter((f) => - level === 1 ? f.__typename !== next.__typename : true + const newFilters = [...filters].filter( + (f) => + f.__typename !== "DataCubeAbout" && + (level === 1 ? f.__typename !== next.__typename : true) ); if (level === 1) { @@ -353,7 +355,6 @@ const NavItem = ({ Boolean ) ); - // const nextIndex = filters.findIndex((f) => f.iri === next.iri); const newFilters = filters.filter((d) => d.iri !== next.iri); return `/browse/${newFilters From bde860c7feb4a783b24068e212b002797fedbfe7 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 17:17:40 +0200 Subject: [PATCH 21/48] fix: Do not clear the cubes when new query is being executed --- app/browser/select-dataset-step.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/browser/select-dataset-step.tsx b/app/browser/select-dataset-step.tsx index 53d19aaac..6559d1b6c 100644 --- a/app/browser/select-dataset-step.tsx +++ b/app/browser/select-dataset-step.tsx @@ -177,7 +177,7 @@ const SelectDatasetStepContent = () => { }); const { allCubes, cubes } = React.useMemo(() => { - if (fetching || error || (data && data.searchCubes.length === 0) || !data) { + if ((data && data.searchCubes.length === 0) || !data) { return { allCubes: [], cubes: [], @@ -203,7 +203,7 @@ const SelectDatasetStepContent = () => { return d.cube.subthemes.some((d) => subthemes.includes(d.iri)); }), }; - }, [data, error, fetching, filters]); + }, [data, filters]); if (configState.state !== "SELECTING_DATASET") { return null; From d1eff4531a8eb379c4f3e6d09a36a91669ff3981 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 17:41:49 +0200 Subject: [PATCH 22/48] fix: Only update browse state if router is ready ...to prevent overwriting of parameters. --- app/browser/context.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/browser/context.tsx b/app/browser/context.tsx index 3222beb6c..6539581e7 100644 --- a/app/browser/context.tsx +++ b/app/browser/context.tsx @@ -87,15 +87,19 @@ const useQueryParamsState = ( : undefined; const dataset = extractParamFromPath(router.asPath, "dataset"); const query = sp ? Object.fromEntries(sp.entries()) : undefined; + if (dataset && query && !query.dataset) { query.dataset = dataset[0]; } + return query ? parse(query) : initialState; }); useEffect(() => { - rawSetState(parse(router.query)); - }, [parse, router.query]); + if (router.isReady) { + rawSetState(parse(router.query)); + } + }, [parse, router.isReady, router.query]); const setState = useEvent((stateUpdate: T) => { rawSetState((curState) => { From 4f65f10fc1aa5096e242ff9bcb4977ae8d717b22 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Thu, 26 Oct 2023 18:04:41 +0200 Subject: [PATCH 23/48] docs: Update CHANGELOG --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e222f0b0..bca18ba30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,17 @@ You can also check the [release page](https://github.com/visualize-admin/visuali - Features - It's now possible to adjust Combo charts colors 🧑‍🎨 + - Subthemes are now displayed even when organization section is the second one (Browse page) - Fixes - Color picker in now in sync with selected color palette + - Dataset counts should now be correct in all cases + - Dataset tags now wrap + - It's now possible to de-select only one filter when both themes and organizations are filtered (Browse page) + - Search string is now correctly persisted in search box when refreshing the page (Browse page) + - Fixed issue with two queries being sent when refreshing the page when search string was entered (Browse page) + - Fixed issue with filtering / unfiltering subthemes that resulted in 404 error (Browse page) - Performance + - Improved performance of searching for and retrieving datasets (Browse page) - Improved the performance of data download - (min|max)Inclusive values stored in `sh:or` are now also retrieved - Style From a726792196b82cb9999333edee8b8d71b74d2feb Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 09:21:24 +0200 Subject: [PATCH 24/48] fix: Include subthemes when calculating cube search scores --- app/rdf/query-search-score-utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index f604ec14a..55115aea1 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -11,6 +11,7 @@ export const weights = { themeLabel: 1, publisher: 1, creatorLabel: 1, + subthemeLabel: 1, }; export const exactMatchPoints = weights.title * 2; @@ -60,6 +61,7 @@ export const computeScores = ( infoPerCube[cube.iri] = { score }; } } + for (const k of Object.keys(infoPerCube)) { if (infoPerCube[k]?.score === 0) { delete infoPerCube[k]; From 95341b1924663d55b328b390e8978a7aadb7e901 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 11:42:54 +0200 Subject: [PATCH 25/48] perf: Specify graph when querying themes --- app/rdf/query-search.ts | 54 +++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 93774746b..ff4f2a0f0 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -17,25 +17,6 @@ import { pragmas } from "./create-source"; import { computeScores, highlight } from "./query-search-score-utils"; import { makeVisualizeDatasetFilter } from "./query-utils"; -const toNamedNode = (x: string) => { - return `<${x}>`; -}; - -const makeInFilter = (varName: string, values: string[]) => { - return ` - ${ - values.length > 0 - ? `FILTER (bound(?${varName}) && ?${varName} IN (${values.map( - toNamedNode - )}))` - : "" - }`; -}; - -const icontains = (left: string, right: string) => { - return `CONTAINS(LCASE(${left}), LCASE("${right}"))`; -}; - // Keep in sync with the query. type RawSearchCube = { iri: NamedNode; @@ -107,6 +88,21 @@ const buildLocalizedSubQuery = ( `; }; +const makeInFilter = (name: string, values: string[]) => { + return ` + ${ + values.length > 0 + ? `FILTER (bound(?${name}) && ?${name} IN (${values.map( + (d) => `<${d}>` + )}))` + : "" + }`; +}; + +const icontains = (left: string, right: string) => { + return `CONTAINS(LCASE(${left}), LCASE("${right}"))`; +}; + export const searchCubes = async ({ query, locale: _locale, @@ -147,12 +143,21 @@ export const searchCubes = async ({ ?iri ${ns.schema.datePublished} ?datePublished . FILTER(DATATYPE(?datePublished) = ${ns.xsd.date}) } + OPTIONAL { ?iri ${ns.dcat.theme} ?themeIri . - ${buildLocalizedSubQuery("themeIri", "schema:name", "themeLabel", { - locale, - })} + GRAPH { + ?themeIri a ${ns.schema.DefinedTerm} ; + ${ + ns.schema.inDefinedTermSet + } . + ${buildLocalizedSubQuery("themeIri", "schema:name", "themeLabel", { + locale, + })} + } } + ${makeInFilter("themeIri", themeValues)} + OPTIONAL { ?iri ${ns.schema.about} ?subthemeIri . OPTIONAL { @@ -171,8 +176,7 @@ export const searchCubes = async ({ includeDrafts: !!includeDrafts, cubeIriVar: "?iri", })} - ${makeInFilter("creatorIri", creatorValues)} - ${makeInFilter("themeIri", themeValues)} + OPTIONAL { ?iri ${ns.dcterms.creator} ?creatorIri . GRAPH { @@ -188,6 +192,8 @@ export const searchCubes = async ({ )} } } + ${makeInFilter("creatorIri", creatorValues)} + ${ query ? `FILTER( From 986e8423bc7da8c65e05d33f83596038c5208b72 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 11:43:35 +0200 Subject: [PATCH 26/48] perf: Specify graph for subthemes --- app/rdf/query-search.ts | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index ff4f2a0f0..40853bd0d 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -158,24 +158,23 @@ export const searchCubes = async ({ } ${makeInFilter("themeIri", themeValues)} + VALUES (?subthemeGraph ?subthemeTermset) { + # Add more subtheme termsets here when they are available + ( ) + } OPTIONAL { ?iri ${ns.schema.about} ?subthemeIri . - OPTIONAL { - ?subthemeIri ${ - ns.schema.inDefinedTermSet - } . + GRAPH ?subthemeGraph { + ?subthemeIri a ${ns.schema.DefinedTerm} ; + ${ns.schema.inDefinedTermSet} ?subthemeTermset . + ${buildLocalizedSubQuery( + "subthemeIri", + "schema:name", + "subthemeLabel", + { locale } + )} } - ${buildLocalizedSubQuery( - "subthemeIri", - "schema:name", - "subthemeLabel", - { locale } - )} } - ${makeVisualizeDatasetFilter({ - includeDrafts: !!includeDrafts, - cubeIriVar: "?iri", - })} OPTIONAL { ?iri ${ns.dcterms.creator} ?creatorIri . @@ -194,6 +193,11 @@ export const searchCubes = async ({ } ${makeInFilter("creatorIri", creatorValues)} + ${makeVisualizeDatasetFilter({ + includeDrafts: !!includeDrafts, + cubeIriVar: "?iri", + })} + ${ query ? `FILTER( From 7b9c5084d1d31e69df55a46f53f51518a23376c7 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 11:44:01 +0200 Subject: [PATCH 27/48] chore: Comment out published date datatype condition for now --- app/rdf/query-search.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 40853bd0d..c0b051cc7 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -141,7 +141,8 @@ export const searchCubes = async ({ OPTIONAL { ?iri ${ns.schema.creativeWorkStatus} ?status . } OPTIONAL { ?iri ${ns.schema.datePublished} ?datePublished . - FILTER(DATATYPE(?datePublished) = ${ns.xsd.date}) + # Uncomment once it's valid for all cubes, see https://github.com/visualize-admin/visualization-tool/pull/1236#issuecomment-1781498261 + # FILTER(DATATYPE(?datePublished) = ${ns.xsd.date}) } OPTIONAL { From 4b9e50efeb94289011e3be69d86b12b3ce3e4fc6 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 12:03:48 +0200 Subject: [PATCH 28/48] fix: Keep multi-lang filtering --- app/rdf/query-search-score-utils.ts | 8 +- app/rdf/query-search.ts | 115 ++++++++++++++++------------ 2 files changed, 66 insertions(+), 57 deletions(-) diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index 55115aea1..c40ba7075 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -30,7 +30,7 @@ export const computeScores = ( if (query) { for (const cube of cubes) { - let score = 0; + let score = 1; for (const [field, weight] of Object.entries(weights) as [ keyof typeof weights, @@ -61,12 +61,6 @@ export const computeScores = ( infoPerCube[cube.iri] = { score }; } } - - for (const k of Object.keys(infoPerCube)) { - if (infoPerCube[k]?.score === 0) { - delete infoPerCube[k]; - } - } } else { for (const cube of cubes) { infoPerCube[cube.iri] = { score: 1 }; diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index c0b051cc7..9e487bc8f 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -145,6 +145,23 @@ export const searchCubes = async ({ # FILTER(DATATYPE(?datePublished) = ${ns.xsd.date}) } + OPTIONAL { + ?iri ${ns.dcterms.creator} ?creatorIri . + GRAPH { + ?creatorIri a ${ns.schema.Organization} ; + ${ + ns.schema.inDefinedTermSet + } . + ${buildLocalizedSubQuery( + "creatorIri", + "schema:name", + "creatorLabel", + { locale } + )} + } + } + ${makeInFilter("creatorIri", creatorValues)} + OPTIONAL { ?iri ${ns.dcat.theme} ?themeIri . GRAPH { @@ -177,61 +194,59 @@ export const searchCubes = async ({ } } - OPTIONAL { - ?iri ${ns.dcterms.creator} ?creatorIri . - GRAPH { - ?creatorIri a ${ns.schema.Organization} ; - ${ - ns.schema.inDefinedTermSet - } . - ${buildLocalizedSubQuery( - "creatorIri", - "schema:name", - "creatorLabel", - { locale } - )} - } - } - ${makeInFilter("creatorIri", creatorValues)} - ${makeVisualizeDatasetFilter({ includeDrafts: !!includeDrafts, cubeIriVar: "?iri", })} - ${ - query - ? `FILTER( - ${query - ?.split(" ") - .slice(0, 1) - .map( - (x) => - `${icontains("?title", x)} || ${icontains("?description", x)}` - ) - .join(" || ")} - - || (bound(?publisher) && ${query - .split(" ") - .map((x) => icontains("?publisher", x)) - .join(" || ")}) - - || (bound(?themeLabel) && ${query - .split(" ") - .map((x) => icontains("?themeLabel", x)) - .join(" || ")}) - - || (bound(?subthemeLabel) && ${query - .split(" ") - .map((x) => icontains("?subthemeLabel", x)) - .join(" || ")}) - - || (bound(?creatorLabel) && ${query - .split(" ") - .map((x) => icontains("?creatorLabel", x)) - .join(" || ")}) - )` - : "" + { + ?iri ${ns.schema.name} ?_title . + BIND(LANG(?_title) AS ?_lang) . + OPTIONAL { + ?iri ${ns.schema.description} ?_description . + FILTER(LANG(?_description) = ?_lang) + } + OPTIONAL { + ?iri ${ns.dcterms.creator}/${ns.schema.name} ?_creatorLabel . + FILTER(LANG(?_creatorLabel) = ?_lang) + } + OPTIONAL { + ?iri ${ns.dcat.theme}/${ns.schema.name} ?_themeLabel . + FILTER(LANG(?_themeLabel) = ?_lang) + } + OPTIONAL { + ?iri ${ns.schema.about}/${ns.schema.name} ?_subthemeLabel . + FILTER(LANG(?_subthemeLabel) = ?_lang) + } + OPTIONAL { + ?iri ${ns.dcterms.publisher}/${ns.schema.name} ?_publisher . + FILTER(LANG(?_publisher) = ?_lang) + } + ${ + query + ? ` + FILTER( + ${query + .split(" ") + .map( + (d) => + `${icontains("?_title", d)} || ${icontains( + "?_description", + d + )} || ${icontains("?_creatorLabel", d)} || ${icontains( + "?_themeLabel", + d + )} || ${icontains("?_subthemeLabel", d)} || ${icontains( + "?_publisher", + d + )}` + ) + .join(" || ")} + ) + ` + : "" + } + } ` .ORDER() From d16b2b983d57a53c2db499633ce3a9f97c33a2e9 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 12:11:15 +0200 Subject: [PATCH 29/48] fix: Tests --- app/rdf/query-search-score-utils.ts | 1 + app/rdf/query-search.spec.ts | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index c40ba7075..e4f929147 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -30,6 +30,7 @@ export const computeScores = ( if (query) { for (const cube of cubes) { + // If a cube have been found, it has at least a score of 1. let score = 1; for (const [field, weight] of Object.entries(weights) as [ diff --git a/app/rdf/query-search.spec.ts b/app/rdf/query-search.spec.ts index 06db00f04..c4b263ce9 100644 --- a/app/rdf/query-search.spec.ts +++ b/app/rdf/query-search.spec.ts @@ -8,7 +8,7 @@ import { jest.mock("@tpluscode/sparql-builder", () => ({})); describe("compute scores", () => { - const scores = [ + const cubes = [ { iri: "a", title: "national" }, { iri: "b", title: "national", description: "economy" }, { iri: "c", creatorLabel: "national" }, @@ -17,11 +17,11 @@ describe("compute scores", () => { ] as unknown as ParsedRawSearchCube[]; it("should compute weighted score per cube from score rows", () => { - const reduced = computeScores(scores, { query: "national economy" }); - expect(reduced.a.score).toEqual(weights.title); - expect(reduced.b.score).toEqual(weights.title + weights.description); - expect(reduced.c.score).toEqual(weights.creatorLabel); - expect(reduced.d).toBeUndefined(); - expect(reduced.e.score).toEqual(weights.title * 2 + exactMatchPoints); + const scores = computeScores(cubes, { query: "national economy" }); + expect(scores.a.score).toEqual(1 + weights.title); + expect(scores.b.score).toEqual(1 + weights.title + weights.description); + expect(scores.c.score).toEqual(1 + weights.creatorLabel); + expect(scores.d.score).toEqual(1); + expect(scores.e.score).toEqual(1 + weights.title * 2 + exactMatchPoints); }); }); From 6b1cedc9ca181bdc73b6e78de2b54e9d2403783d Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 12:33:04 +0200 Subject: [PATCH 30/48] test: Add more E2E search tests --- e2e/search.spec.ts | 90 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/e2e/search.spec.ts b/e2e/search.spec.ts index a4c9c3d97..7cc6c8852 100644 --- a/e2e/search.spec.ts +++ b/e2e/search.spec.ts @@ -1,7 +1,8 @@ import { within } from "@playwright-testing-library/test"; -import { Locator } from "@playwright/test"; +import { Locator, Page } from "@playwright/test"; import { setup } from "./common"; +import { Selectors } from "./selectors"; const { test, expect } = setup(); @@ -45,7 +46,7 @@ test("search results count coherence", async ({ page, selectors }) => { "Swiss Federal Office of Energy SFOE", ]; - for (let t of [...categories, ...themes]) { + for (const t of [...categories, ...themes]) { await page.goto("/en/browse?dataSource=Int"); await selectors.search.resultsCount(); @@ -121,3 +122,88 @@ test("sort order", async ({ page, selectors, screen, actions }) => { await page.keyboard.press("Enter"); expect(await getSelectValue(select)).toBe("SCORE"); }); + +const getResultCountForSearch = async ( + search: string, + { + page, + selectors, + locale, + }: { page: Page; selectors: Selectors; locale: string } +) => { + await page.goto( + `/${locale}/browse?search=${encodeURIComponent(search)}&dataSource=Prod` + ); + const resultCount = await selectors.search.resultsCount(); + const count = (await resultCount.textContent())?.split(" ")[0]; + expect(count).toBeTruthy(); + + return count; +}; + +test("sort order consistency", async ({ page, selectors }) => { + const count1 = await getResultCountForSearch("wasser wald", { + locale: "en", + page, + selectors, + }); + const count2 = await getResultCountForSearch("wald wasser", { + locale: "en", + page, + selectors, + }); + + expect(count1).toEqual(count2); +}); + +test("sort language consistency", async ({ page, selectors }) => { + const count1 = await getResultCountForSearch("wasser", { + locale: "en", + page, + selectors, + }); + const count2 = await getResultCountForSearch("wasser", { + locale: "de", + page, + selectors, + }); + const count3 = await getResultCountForSearch("wasser", { + locale: "fr", + page, + selectors, + }); + const count4 = await getResultCountForSearch("wasser", { + locale: "it", + page, + selectors, + }); + + expect(count1).toEqual(count2); + expect(count1).toEqual(count3); + expect(count1).toEqual(count4); + + const count5 = await getResultCountForSearch("water", { + locale: "en", + page, + selectors, + }); + const count6 = await getResultCountForSearch("eaux", { + locale: "de", + page, + selectors, + }); + const count7 = await getResultCountForSearch("acque", { + locale: "fr", + page, + selectors, + }); + const count8 = await getResultCountForSearch("water", { + locale: "it", + page, + selectors, + }); + + expect(count5).toEqual(count6); + expect(count5).toEqual(count7); + expect(count5).toEqual(count8); +}); From 41353f6ae422c71052cb1b295bc32a9f7b701965 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 12:51:13 +0200 Subject: [PATCH 31/48] fix: creativeWorkStatus has to be set --- app/rdf/query-search.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 9e487bc8f..7ac1a387c 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -138,7 +138,7 @@ export const searchCubes = async ({ })} OPTIONAL { ?versionHistory ${ns.schema.hasPart} ?iri . } OPTIONAL { ?iri ${ns.dcterms.publisher} ?publisher . } - OPTIONAL { ?iri ${ns.schema.creativeWorkStatus} ?status . } + ?iri ${ns.schema.creativeWorkStatus} ?status . OPTIONAL { ?iri ${ns.schema.datePublished} ?datePublished . # Uncomment once it's valid for all cubes, see https://github.com/visualize-admin/visualization-tool/pull/1236#issuecomment-1781498261 From c02a7d2c2d743a0507e68f2047181ce0a98ea110 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 13:01:39 +0200 Subject: [PATCH 32/48] style: Animate filters title --- app/browser/select-dataset-step.tsx | 49 ++++++++++++++++++++++------- app/components/presence.tsx | 2 +- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/app/browser/select-dataset-step.tsx b/app/browser/select-dataset-step.tsx index 6559d1b6c..b895f5f73 100644 --- a/app/browser/select-dataset-step.tsx +++ b/app/browser/select-dataset-step.tsx @@ -22,6 +22,7 @@ import { BANNER_HEIGHT, BANNER_MARGIN_TOP, bannerPresenceProps, + DURATION, MotionBox, navPresenceProps, smoothPresenceProps, @@ -111,7 +112,6 @@ const useStyles = makeStyles((theme) => ({ }, filters: { display: "block", - marginBottom: theme.spacing(4), color: theme.palette.grey[800], }, })); @@ -296,16 +296,43 @@ const SelectDatasetStepContent = () => { ) : ( - {filters.length > 0 && ( - - {queryFilters.map((d) => d.label).join(", ")} - - )} - + + {queryFilters.length > 0 && ( + + + {queryFilters.map((d) => d.label).join(", ")} + + + )} + Date: Fri, 27 Oct 2023 13:52:01 +0200 Subject: [PATCH 33/48] perf: Do not fire cubes, themes and organizations queries on dataset preview page --- CHANGELOG.md | 1 + app/browser/context.tsx | 31 ++++++++++++++++------------- app/browser/dataset-browse.tsx | 4 +++- app/browser/select-dataset-step.tsx | 1 + 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bca18ba30..9b9444add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ You can also check the [release page](https://github.com/visualize-admin/visuali - Fixed issue with filtering / unfiltering subthemes that resulted in 404 error (Browse page) - Performance - Improved performance of searching for and retrieving datasets (Browse page) + - Cubes, themes and organizations queries aren't fired anymore when previewing a dataset - Improved the performance of data download - (min|max)Inclusive values stored in `sh:or` are now also retrieved - Style diff --git a/app/browser/context.tsx b/app/browser/context.tsx index 6539581e7..452f771c1 100644 --- a/app/browser/context.tsx +++ b/app/browser/context.tsx @@ -117,20 +117,6 @@ export const useBrowseState = () => { const { dataSource } = useDataSourceStore(); const locale = useLocale(); const inputRef = useRef(null); - const [{ data: themeData }] = useThemesQuery({ - variables: { - sourceType: dataSource.type, - sourceUrl: dataSource.url, - locale, - }, - }); - const [{ data: orgData }] = useOrganizationsQuery({ - variables: { - sourceType: dataSource.type, - sourceUrl: dataSource.url, - locale, - }, - }); const [browseParams, setParams] = useQueryParamsState( {}, @@ -149,6 +135,23 @@ export const useBrowseState = () => { dataset: paramDataset, } = browseParams; + const [{ data: themeData }] = useThemesQuery({ + variables: { + sourceType: dataSource.type, + sourceUrl: dataSource.url, + locale, + }, + pause: !!paramDataset, + }); + const [{ data: orgData }] = useOrganizationsQuery({ + variables: { + sourceType: dataSource.type, + sourceUrl: dataSource.url, + locale, + }, + pause: !!paramDataset, + }); + // Support /browse?dataset= and legacy /browse/dataset/ const dataset = type === "dataset" ? iri : paramDataset; const filters = getFiltersFromParams(browseParams, { diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index 610e1bd14..6d25f1f4b 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -596,13 +596,14 @@ const NavSection = ({ export const SearchFilters = ({ cubes }: { cubes: SearchCubeResult[] }) => { const { dataSource } = useDataSourceStore(); const locale = useLocale(); - const { filters } = useBrowseContext(); + const { filters, dataset } = useBrowseContext(); const [{ data: allThemes }] = useThemesQuery({ variables: { sourceType: dataSource.type, sourceUrl: dataSource.url, locale, }, + pause: !!dataset, }); const [{ data: allOrgs }] = useOrganizationsQuery({ variables: { @@ -610,6 +611,7 @@ export const SearchFilters = ({ cubes }: { cubes: SearchCubeResult[] }) => { sourceUrl: dataSource.url, locale, }, + pause: !!dataset, }); const counts = useMemo(() => { diff --git a/app/browser/select-dataset-step.tsx b/app/browser/select-dataset-step.tsx index b895f5f73..85e18ae6d 100644 --- a/app/browser/select-dataset-step.tsx +++ b/app/browser/select-dataset-step.tsx @@ -169,6 +169,7 @@ const SelectDatasetStepContent = () => { includeDrafts, filters: queryFilters, }, + pause: !!dataset, }); useRedirectToVersionedCube({ From e2de8ca6e4dd1b701b2c17699f99582358a54d40 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Fri, 27 Oct 2023 14:17:35 +0200 Subject: [PATCH 34/48] fix: Do not filter publishers by language --- app/rdf/query-search.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 7ac1a387c..02adb49d8 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -218,10 +218,7 @@ export const searchCubes = async ({ ?iri ${ns.schema.about}/${ns.schema.name} ?_subthemeLabel . FILTER(LANG(?_subthemeLabel) = ?_lang) } - OPTIONAL { - ?iri ${ns.dcterms.publisher}/${ns.schema.name} ?_publisher . - FILTER(LANG(?_publisher) = ?_lang) - } + ${ query ? ` From 3ed4efc62026a0237796ec74a79f3d7429013d76 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Mon, 30 Oct 2023 09:11:45 +0100 Subject: [PATCH 35/48] perf: Further optimize cube search query --- app/rdf/query-search.ts | 49 ++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 02adb49d8..c74b40bea 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -99,10 +99,6 @@ const makeInFilter = (name: string, values: string[]) => { }`; }; -const icontains = (left: string, right: string) => { - return `CONTAINS(LCASE(${left}), LCASE("${right}"))`; -}; - export const searchCubes = async ({ query, locale: _locale, @@ -139,11 +135,7 @@ export const searchCubes = async ({ OPTIONAL { ?versionHistory ${ns.schema.hasPart} ?iri . } OPTIONAL { ?iri ${ns.dcterms.publisher} ?publisher . } ?iri ${ns.schema.creativeWorkStatus} ?status . - OPTIONAL { - ?iri ${ns.schema.datePublished} ?datePublished . - # Uncomment once it's valid for all cubes, see https://github.com/visualize-admin/visualization-tool/pull/1236#issuecomment-1781498261 - # FILTER(DATATYPE(?datePublished) = ${ns.xsd.date}) - } + OPTIONAL { ?iri ${ns.schema.datePublished} ?datePublished . } OPTIONAL { ?iri ${ns.dcterms.creator} ?creatorIri . @@ -176,10 +168,14 @@ export const searchCubes = async ({ } ${makeInFilter("themeIri", themeValues)} - VALUES (?subthemeGraph ?subthemeTermset) { - # Add more subtheme termsets here when they are available - ( ) - } + # Add more subtheme termsets here when they are available + ${ + creatorValues.includes( + "https://register.ld.admin.ch/opendataswiss/org/bundesamt-fur-umwelt-bafu" + ) + ? "VALUES (?subthemeGraph ?subthemeTermset) { ( ) }" + : "" + } OPTIONAL { ?iri ${ns.schema.about} ?subthemeIri . GRAPH ?subthemeGraph { @@ -222,28 +218,17 @@ export const searchCubes = async ({ ${ query ? ` + VALUES ?keyword { ${query.split(" ").map((d) => `"${d}"`)} } FILTER( - ${query - .split(" ") - .map( - (d) => - `${icontains("?_title", d)} || ${icontains( - "?_description", - d - )} || ${icontains("?_creatorLabel", d)} || ${icontains( - "?_themeLabel", - d - )} || ${icontains("?_subthemeLabel", d)} || ${icontains( - "?_publisher", - d - )}` - ) - .join(" || ")} - ) - ` + CONTAINS(LCASE(?_title), LCASE(?keyword)) || + CONTAINS(LCASE(?_description), LCASE(?keyword)) || + CONTAINS(LCASE(?_creatorLabel), LCASE(?keyword)) || + CONTAINS(LCASE(?_themeLabel), LCASE(?keyword)) || + CONTAINS(LCASE(?_subthemeLabel), LCASE(?keyword)) || + CONTAINS(LCASE(?_publisher), LCASE(?keyword)) + )` : "" } - } ` .ORDER() From a01b3f718c8071db4b180458e5c6ac7f46456a79 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Mon, 30 Oct 2023 09:24:02 +0100 Subject: [PATCH 36/48] feat: Improve GQLDebugPanel --- CHANGELOG.md | 1 + app/gql-flamegraph/devtool.tsx | 104 ++++++++++++++++++++++++--------- 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b9444add..a57cfc981 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ You can also check the [release page](https://github.com/visualize-admin/visuali - Map now outlines shapes on hover, instead of changing their colors - Maintenance - Added retrieval of dimension units via `qudt:hasUnit` (but kept `qudt:unit` for backward compatibility) + - Improved GQL debug panel (added resolver variables and rectangles to visually indicate resolving times) # [3.23.0] - 2023-10-17 diff --git a/app/gql-flamegraph/devtool.tsx b/app/gql-flamegraph/devtool.tsx index 03e81fc2e..1d814264b 100644 --- a/app/gql-flamegraph/devtool.tsx +++ b/app/gql-flamegraph/devtool.tsx @@ -182,12 +182,14 @@ const AccordionOperation = ({ operation, start, end, + maxTime, ...accordionProps }: { result: VisualizeOperationResult | undefined; operation: Operation; start: number; end: number; + maxTime: number; } & Omit) => { const duration = useMemo(() => { const all = result?.extensions?.timings @@ -215,11 +217,19 @@ const AccordionOperation = ({ }} > - {operation.key} {getOperationQueryName(operation)} - - {result ? "✅" : "🔄"} - + {getOperationQueryName(operation)} +
+ + + +
{duration}ms {accordionProps.expanded ? ( - - +
+ + Variables + +
+ {result?.operation.variables && + Object.entries(result.operation.variables).map(([k, v]) => { + return ( + + + {k}: {JSON.stringify(v, null, 2)} + + + ); + })} +
+
+ +
Resolvers - <> - {result?.extensions?.timings ? ( +
+ {result?.extensions?.timings && ( - ) : null} - - -
+ )} +
+
+
SPARQL queries ({result?.extensions?.queries.length}) @@ -288,7 +327,7 @@ const Queries = ({ queries }: { queries: RequestQueryMeta[] }) => { {queries.map((q, i) => { const text = q.text.replace(/\n\n/gm, "\n"); return ( -
+
{q.endTime - q.startTime}ms {" "} @@ -383,6 +422,16 @@ function GqlDebug({ controller }: { controller: GraphqlOperationsController }) { return null; } + const preparedResults = sortBy(results, (r) => + opsStartMap.get(r.operation.key) + ).filter((x) => x?.extensions?.timings); + const maxOperationTime = Math.max( + ...preparedResults.map((r) => { + const timings = flatten(r.extensions.timings); + return Math.max(...timings.map((x) => x.end - x.start)); + }) + ); + return (
@@ -391,21 +440,20 @@ function GqlDebug({ controller }: { controller: GraphqlOperationsController }) { - {sortBy(results, (r) => opsStartMap.get(r.operation.key)) - .filter((x) => x?.extensions?.timings) - .map((result, i) => ( - - setExpandedId(expanded ? result.operation.key : undefined) - } - /> - ))} + {preparedResults.map((result, i) => ( + + setExpandedId(expanded ? result.operation.key : undefined) + } + maxTime={maxOperationTime} + /> + ))}
); From e90b2a21b052932e28403e399d7bc67218105563 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Mon, 30 Oct 2023 09:54:54 +0100 Subject: [PATCH 37/48] fix: Search query keywords --- app/rdf/query-search.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index c74b40bea..991206396 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -218,7 +218,10 @@ export const searchCubes = async ({ ${ query ? ` - VALUES ?keyword { ${query.split(" ").map((d) => `"${d}"`)} } + VALUES ?keyword { ${query + .split(" ") + .map((d) => `"${d}"`) + .join(" ")} } FILTER( CONTAINS(LCASE(?_title), LCASE(?keyword)) || CONTAINS(LCASE(?_description), LCASE(?keyword)) || From b4e87bc391c4f76c50d31ff292510f4fe12a3797 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Mon, 30 Oct 2023 13:03:42 +0100 Subject: [PATCH 38/48] fix: Copy button placement --- app/gql-flamegraph/devtool.tsx | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/app/gql-flamegraph/devtool.tsx b/app/gql-flamegraph/devtool.tsx index 1d814264b..4a353b020 100644 --- a/app/gql-flamegraph/devtool.tsx +++ b/app/gql-flamegraph/devtool.tsx @@ -328,13 +328,15 @@ const Queries = ({ queries }: { queries: RequestQueryMeta[] }) => { const text = q.text.replace(/\n\n/gm, "\n"); return (
- - {q.endTime - q.startTime}ms - {" "} - -{" "} - - Copy - +
+ + {q.endTime - q.startTime}ms + {" "} + -{" "} + + Copy + +
Date: Mon, 30 Oct 2023 13:26:33 +0100 Subject: [PATCH 39/48] fix: Show published and draft versions of cubes with the same version history / iri --- app/rdf/query-search.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 991206396..0228e5b39 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -254,7 +254,8 @@ export const searchCubes = async ({ const cubes = rawCubes .map((cube) => { const versionHistory = versionHistoryPerCube[cube.iri]; - const dedupIdentifier = versionHistory ?? cube.iri; + // Need to keep both published and draft cubes with the same iri. + const dedupIdentifier = (versionHistory ?? cube.iri) + cube.status; if (seenCubes.has(dedupIdentifier)) { return null; From 8d87a8278c9e3efa077ad74c697da52c88d54a2d Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Mon, 30 Oct 2023 13:26:36 +0100 Subject: [PATCH 40/48] perf: Optimize search query --- app/rdf/query-search.ts | 80 ++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 0228e5b39..d116bdaf1 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -195,44 +195,50 @@ export const searchCubes = async ({ cubeIriVar: "?iri", })} - { - ?iri ${ns.schema.name} ?_title . - BIND(LANG(?_title) AS ?_lang) . - OPTIONAL { - ?iri ${ns.schema.description} ?_description . - FILTER(LANG(?_description) = ?_lang) - } - OPTIONAL { - ?iri ${ns.dcterms.creator}/${ns.schema.name} ?_creatorLabel . - FILTER(LANG(?_creatorLabel) = ?_lang) - } - OPTIONAL { - ?iri ${ns.dcat.theme}/${ns.schema.name} ?_themeLabel . - FILTER(LANG(?_themeLabel) = ?_lang) - } - OPTIONAL { - ?iri ${ns.schema.about}/${ns.schema.name} ?_subthemeLabel . - FILTER(LANG(?_subthemeLabel) = ?_lang) - } - - ${ - query - ? ` - VALUES ?keyword { ${query - .split(" ") - .map((d) => `"${d}"`) - .join(" ")} } - FILTER( - CONTAINS(LCASE(?_title), LCASE(?keyword)) || - CONTAINS(LCASE(?_description), LCASE(?keyword)) || - CONTAINS(LCASE(?_creatorLabel), LCASE(?keyword)) || - CONTAINS(LCASE(?_themeLabel), LCASE(?keyword)) || - CONTAINS(LCASE(?_subthemeLabel), LCASE(?keyword)) || - CONTAINS(LCASE(?_publisher), LCASE(?keyword)) - )` - : "" - } + ${ + query + ? ` + VALUES ?keyword {${query + .split(" ") + .map((d) => `"${d}"`) + .join(" ")}} + + FILTER( + CONTAINS(LCASE(?title_de), LCASE(?keyword)) || + CONTAINS(LCASE(?title_fr), LCASE(?keyword)) || + CONTAINS(LCASE(?title_it), LCASE(?keyword)) || + CONTAINS(LCASE(?title_en), LCASE(?keyword)) || + CONTAINS(LCASE(?title_), LCASE(?keyword)) || + + CONTAINS(LCASE(?description_de), LCASE(?keyword)) || + CONTAINS(LCASE(?description_fr), LCASE(?keyword)) || + CONTAINS(LCASE(?description_it), LCASE(?keyword)) || + CONTAINS(LCASE(?description_en), LCASE(?keyword)) || + CONTAINS(LCASE(?description_), LCASE(?keyword)) || + + CONTAINS(LCASE(?creatorLabel_de), LCASE(?keyword)) || + CONTAINS(LCASE(?creatorLabel_fr), LCASE(?keyword)) || + CONTAINS(LCASE(?creatorLabel_it), LCASE(?keyword)) || + CONTAINS(LCASE(?creatorLabel_en), LCASE(?keyword)) || + CONTAINS(LCASE(?creatorLabel_), LCASE(?keyword)) || + + CONTAINS(LCASE(?themeLabel_de), LCASE(?keyword)) || + CONTAINS(LCASE(?themeLabel_fr), LCASE(?keyword)) || + CONTAINS(LCASE(?themeLabel_it), LCASE(?keyword)) || + CONTAINS(LCASE(?themeLabel_en), LCASE(?keyword)) || + CONTAINS(LCASE(?themeLabel_), LCASE(?keyword)) || + + CONTAINS(LCASE(?subthemeLabel_de), LCASE(?keyword)) || + CONTAINS(LCASE(?subthemeLabel_fr), LCASE(?keyword)) || + CONTAINS(LCASE(?subthemeLabel_it), LCASE(?keyword)) || + CONTAINS(LCASE(?subthemeLabel_en), LCASE(?keyword)) || + CONTAINS(LCASE(?subthemeLabel_), LCASE(?keyword)) || + + CONTAINS(LCASE(?publisher), LCASE(?keyword)) + )` + : "" } + ` .ORDER() // Important for the latter part of the query, to return the latest cube per unversioned iri. From 4652385ce6a4713bd6a25f39431bc22e5dbdf3d5 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Mon, 30 Oct 2023 16:15:00 +0100 Subject: [PATCH 41/48] perf: Remove version history-related logic from SearchCubes query --- app/rdf/query-search.ts | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index d116bdaf1..4892f3827 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -1,4 +1,3 @@ -import RDF from "@rdfjs/data-model"; import { SELECT } from "@tpluscode/sparql-builder"; import { descending, group } from "d3"; import { Literal, NamedNode } from "rdf-js"; @@ -22,7 +21,6 @@ type RawSearchCube = { iri: NamedNode; title: Literal; description: Literal; - versionHistory: NamedNode; status: NamedNode; datePublished: Literal; creatorIri: NamedNode; @@ -43,7 +41,6 @@ const parseRawSearchCube = (cube: RawSearchCube): ParsedRawSearchCube => { iri: cube.iri.value, title: cube.title.value, description: cube.description?.value, - versionHistory: cube.versionHistory?.value, status: cube.status.value === ns.adminVocabulary("CreativeWorkStatus/Published").value @@ -123,7 +120,7 @@ export const searchCubes = async ({ .map((v) => v.value) ?? []; const scoresQuery = SELECT.DISTINCT` - ?iri ?title ?status ?datePublished ?versionHistory ?description ?publisher ?creatorIri ?creatorLabel ?themeIri ?themeLabel ?subthemeIri ?subthemeLabel + ?iri ?title ?status ?datePublished ?description ?publisher ?creatorIri ?creatorLabel ?themeIri ?themeLabel ?subthemeIri ?subthemeLabel `.WHERE` ?iri a ${ns.cube.Cube} . ${buildLocalizedSubQuery("iri", "schema:name", "title", { @@ -132,7 +129,6 @@ export const searchCubes = async ({ ${buildLocalizedSubQuery("iri", "schema:description", "description", { locale, })} - OPTIONAL { ?versionHistory ${ns.schema.hasPart} ?iri . } OPTIONAL { ?iri ${ns.dcterms.publisher} ?publisher . } ?iri ${ns.schema.creativeWorkStatus} ?status . OPTIONAL { ?iri ${ns.schema.datePublished} ?datePublished . } @@ -239,29 +235,20 @@ export const searchCubes = async ({ : "" } - ` - .ORDER() - // Important for the latter part of the query, to return the latest cube per unversioned iri. - .BY(RDF.variable("versionHistory")) - .THEN.BY(RDF.variable("datePublished"), true) - .THEN.BY(RDF.variable("iri"), true).prologue`${pragmas}`; + `.prologue`${pragmas}`; const scoreResults = await scoresQuery.execute(sparqlClient.query, { operation: "postUrlencoded", }); const rawCubes = (scoreResults as RawSearchCube[]).map(parseRawSearchCube); const rawCubesByIri = group(rawCubes, (d) => d.iri); - const versionHistoryPerCube = Object.fromEntries( - rawCubes.map((d) => [d.iri, d.versionHistory]) - ); const infoByCube = computeScores(rawCubes, { query }); const seenCubes = new Set(); const cubes = rawCubes .map((cube) => { - const versionHistory = versionHistoryPerCube[cube.iri]; // Need to keep both published and draft cubes with the same iri. - const dedupIdentifier = (versionHistory ?? cube.iri) + cube.status; + const dedupIdentifier = cube.iri + cube.status; if (seenCubes.has(dedupIdentifier)) { return null; From e7088145681fbe4d7c636c37135d31f0ebdeba19 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Mon, 30 Oct 2023 17:27:55 +0100 Subject: [PATCH 42/48] perf: Optimize search cube queries by concatenating themes and sub-themes directly in the query --- app/rdf/query-search-score-utils.ts | 6 +- app/rdf/query-search.ts | 120 ++++++++++++---------------- 2 files changed, 53 insertions(+), 73 deletions(-) diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index e4f929147..9dc710cd9 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -8,10 +8,10 @@ export const parseFloatZeroed = (s: string) => { export const weights = { title: 5, description: 2, - themeLabel: 1, - publisher: 1, creatorLabel: 1, - subthemeLabel: 1, + publisher: 1, + themeLabels: 1, + subthemeLabels: 1, }; export const exactMatchPoints = weights.title * 2; diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 4892f3827..a42b7ed2e 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -26,10 +26,10 @@ type RawSearchCube = { creatorIri: NamedNode; creatorLabel: Literal; publisher: NamedNode; - themeIri: NamedNode; - themeLabel: Literal; - subthemeIri: NamedNode; - subthemeLabel: Literal; + themeIris: NamedNode; + themeLabels: Literal; + subthemeIris: NamedNode; + subthemeLabels: Literal; }; export type ParsedRawSearchCube = { @@ -50,10 +50,10 @@ const parseRawSearchCube = (cube: RawSearchCube): ParsedRawSearchCube => { creatorIri: cube.creatorIri?.value, creatorLabel: cube.creatorLabel?.value, publisher: cube.publisher?.value, - themeIri: cube.themeIri?.value, - themeLabel: cube.themeLabel?.value, - subthemeIri: cube.subthemeIri?.value, - subthemeLabel: cube.subthemeLabel?.value, + themeIris: cube.themeIris?.value, + themeLabels: cube.themeLabels?.value, + subthemeIris: cube.subthemeIris?.value, + subthemeLabels: cube.subthemeLabels?.value, }; }; @@ -96,6 +96,8 @@ const makeInFilter = (name: string, values: string[]) => { }`; }; +const GROUP_SEPARATOR = "|||"; + export const searchCubes = async ({ query, locale: _locale, @@ -119,8 +121,10 @@ export const searchCubes = async ({ ?.filter((x) => x.type === "DataCubeOrganization") .map((v) => v.value) ?? []; - const scoresQuery = SELECT.DISTINCT` - ?iri ?title ?status ?datePublished ?description ?publisher ?creatorIri ?creatorLabel ?themeIri ?themeLabel ?subthemeIri ?subthemeLabel + const scoresQuery = SELECT` + ?iri ?title ?status ?datePublished ?description ?publisher ?creatorIri ?creatorLabel + (GROUP_CONCAT(DISTINCT ?themeIri; SEPARATOR="${GROUP_SEPARATOR}") AS ?themeIris) (GROUP_CONCAT(DISTINCT ?themeLabel; SEPARATOR="${GROUP_SEPARATOR}") AS ?themeLabels) + (GROUP_CONCAT(DISTINCT ?subthemeIri; SEPARATOR="${GROUP_SEPARATOR}") AS ?subthemeIris) (GROUP_CONCAT(DISTINCT ?subthemeLabel; SEPARATOR="${GROUP_SEPARATOR}") AS ?subthemeLabels) `.WHERE` ?iri a ${ns.cube.Cube} . ${buildLocalizedSubQuery("iri", "schema:name", "title", { @@ -235,7 +239,9 @@ export const searchCubes = async ({ : "" } - `.prologue`${pragmas}`; + `.GROUP().BY`?iri`.THEN.BY`?title`.THEN.BY`?status`.THEN.BY`?datePublished` + .THEN.BY`?description`.THEN.BY`?publisher`.THEN.BY`?creatorIri`.THEN + .BY`?creatorLabel`.prologue`${pragmas}`; const scoreResults = await scoresQuery.execute(sparqlClient.query, { operation: "postUrlencoded", @@ -262,68 +268,42 @@ export const searchCubes = async ({ return null; } - const parsedCube: SearchCube = { - iri: rawCubes[0].iri, - title: rawCubes[0].title, - description: null, - creator: null, - publicationStatus: rawCubes[0].status as DataCubePublicationStatus, - datePublished: null, - themes: [], - subthemes: [], - }; - - for (const cube of rawCubes) { - if (!parsedCube.iri) { - parsedCube.iri = cube.iri; - } - - if (!parsedCube.title) { - parsedCube.title = cube.title; - } - - if (!parsedCube.description) { - parsedCube.description = cube.description; - } - - if (!parsedCube.creator && cube.creatorIri) { - parsedCube.creator = { - iri: cube.creatorIri, - label: cube.creatorLabel, - }; - } - - if (!parsedCube.datePublished) { - parsedCube.datePublished = cube.datePublished; - } + if (rawCubes.length > 1) { + console.warn(`Found multiple cubes with the same iri: ${cube.iri}`); + } - if (!parsedCube.publicationStatus) { - parsedCube.publicationStatus = - cube.status as DataCubePublicationStatus; - } + const rawCube = rawCubes[0]; - if ( - cube.themeIri && - cube.themeLabel && - !parsedCube.themes.some((d) => d.iri === cube.themeIri) - ) { - parsedCube.themes.push({ - iri: cube.themeIri, - label: cube.themeLabel, - }); - } + const themeIris = rawCube.themeIris.split(GROUP_SEPARATOR); + const themeLabels = rawCube.themeLabels.split(GROUP_SEPARATOR); + const subthemeIris = rawCube.subthemeIris.split(GROUP_SEPARATOR); + const subthemeLabels = rawCube.subthemeLabels.split(GROUP_SEPARATOR); - if ( - cube.subthemeIri && - cube.subthemeLabel && - !parsedCube.subthemes.some((d) => d.iri === cube.subthemeIri) - ) { - parsedCube.subthemes.push({ - iri: cube.subthemeIri, - label: cube.subthemeLabel, - }); - } - } + const parsedCube: SearchCube = { + iri: rawCube.iri, + title: rawCube.title, + description: rawCube.description, + creator: + rawCube.creatorIri && rawCube.creatorLabel + ? { iri: rawCube.creatorIri, label: rawCube.creatorLabel } + : null, + publicationStatus: rawCube.status as DataCubePublicationStatus, + datePublished: rawCube.datePublished, + themes: + themeIris.length === themeLabels.length + ? themeIris.map((iri, i) => ({ + iri, + label: themeLabels[i], + })) + : [], + subthemes: + subthemeIris.length === subthemeLabels.length + ? subthemeIris.map((iri, i) => ({ + iri, + label: subthemeLabels[i], + })) + : [], + }; return parsedCube; }) From d4117a6f568ed6bdd4d4c3ceefd81592511dd108 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Tue, 31 Oct 2023 09:35:45 +0100 Subject: [PATCH 43/48] fix: No results (search cubes) --- app/rdf/query-search.ts | 5 ++++- e2e/search.spec.ts | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index a42b7ed2e..35d5276a8 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -246,7 +246,10 @@ export const searchCubes = async ({ const scoreResults = await scoresQuery.execute(sparqlClient.query, { operation: "postUrlencoded", }); - const rawCubes = (scoreResults as RawSearchCube[]).map(parseRawSearchCube); + const rawCubes = (scoreResults as RawSearchCube[]) + // Filter out cubes without iri, happens due to grouping, when no cubes are found. + .filter((d) => d.iri) + .map(parseRawSearchCube); const rawCubesByIri = group(rawCubes, (d) => d.iri); const infoByCube = computeScores(rawCubes, { query }); diff --git a/e2e/search.spec.ts b/e2e/search.spec.ts index 7cc6c8852..d3501c08c 100644 --- a/e2e/search.spec.ts +++ b/e2e/search.spec.ts @@ -207,3 +207,8 @@ test("sort language consistency", async ({ page, selectors }) => { expect(count5).toEqual(count7); expect(count5).toEqual(count8); }); + +test("no results", async ({ page }) => { + await page.goto("/en/browse?dataSource=Int&search=foo"); + await page.locator(`:text("No results")`).waitFor({ timeout: 10_000 }); +}); From 67c6bdc53e6c61788b5c0b4a790370f867312111 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Tue, 31 Oct 2023 09:43:16 +0100 Subject: [PATCH 44/48] chore: Typo --- app/rdf/query-search-score-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/rdf/query-search-score-utils.ts b/app/rdf/query-search-score-utils.ts index 9dc710cd9..85a97071b 100644 --- a/app/rdf/query-search-score-utils.ts +++ b/app/rdf/query-search-score-utils.ts @@ -30,7 +30,7 @@ export const computeScores = ( if (query) { for (const cube of cubes) { - // If a cube have been found, it has at least a score of 1. + // If a cube has been found, it has at least a score of 1. let score = 1; for (const [field, weight] of Object.entries(weights) as [ From 056c6403c21276db44c3ec241051cee5188bf045 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Tue, 31 Oct 2023 13:50:19 +0100 Subject: [PATCH 45/48] fix: Theme filtering needs to happen in HAVING part of the query --- app/rdf/query-search.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 35d5276a8..114af57b0 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -121,7 +121,7 @@ export const searchCubes = async ({ ?.filter((x) => x.type === "DataCubeOrganization") .map((v) => v.value) ?? []; - const scoresQuery = SELECT` + let scoresQuery = SELECT` ?iri ?title ?status ?datePublished ?description ?publisher ?creatorIri ?creatorLabel (GROUP_CONCAT(DISTINCT ?themeIri; SEPARATOR="${GROUP_SEPARATOR}") AS ?themeIris) (GROUP_CONCAT(DISTINCT ?themeLabel; SEPARATOR="${GROUP_SEPARATOR}") AS ?themeLabels) (GROUP_CONCAT(DISTINCT ?subthemeIri; SEPARATOR="${GROUP_SEPARATOR}") AS ?subthemeIris) (GROUP_CONCAT(DISTINCT ?subthemeLabel; SEPARATOR="${GROUP_SEPARATOR}") AS ?subthemeLabels) @@ -152,7 +152,7 @@ export const searchCubes = async ({ )} } } - ${makeInFilter("creatorIri", creatorValues)} + ${creatorValues.length ? makeInFilter("creatorIri", creatorValues) : ""} OPTIONAL { ?iri ${ns.dcat.theme} ?themeIri . @@ -166,7 +166,6 @@ export const searchCubes = async ({ })} } } - ${makeInFilter("themeIri", themeValues)} # Add more subtheme termsets here when they are available ${ @@ -238,11 +237,16 @@ export const searchCubes = async ({ )` : "" } - `.GROUP().BY`?iri`.THEN.BY`?title`.THEN.BY`?status`.THEN.BY`?datePublished` .THEN.BY`?description`.THEN.BY`?publisher`.THEN.BY`?creatorIri`.THEN .BY`?creatorLabel`.prologue`${pragmas}`; + if (themeValues.length) { + scoresQuery = scoresQuery.HAVING`${themeValues + .map((iri) => `CONTAINS(LCASE(?themeIris), LCASE("${iri}"))`) + .join(" || ")}` as any; + } + const scoreResults = await scoresQuery.execute(sparqlClient.query, { operation: "postUrlencoded", }); From 7815ab28df522e79785e1629f1a94fae4f198ef1 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Tue, 31 Oct 2023 13:54:19 +0100 Subject: [PATCH 46/48] perf: Do not query themes and organizations separately --- app/browser/context.tsx | 34 +--------- app/browser/dataset-browse.spec.tsx | 24 ++----- app/browser/dataset-browse.tsx | 57 +++++----------- app/browser/filters.tsx | 29 ++------ app/browser/select-dataset-step.tsx | 48 ++++++++++++- app/graphql/queries/data-cubes.graphql | 39 ----------- app/graphql/query-hooks.ts | 94 -------------------------- app/graphql/resolver-types.ts | 28 -------- app/graphql/resolvers/index.ts | 12 ---- app/graphql/resolvers/rdf.ts | 34 ---------- app/graphql/resolvers/sql.ts | 13 ---- app/graphql/schema.graphql | 16 ----- 12 files changed, 76 insertions(+), 352 deletions(-) diff --git a/app/browser/context.tsx b/app/browser/context.tsx index 452f771c1..d09b991bf 100644 --- a/app/browser/context.tsx +++ b/app/browser/context.tsx @@ -7,14 +7,8 @@ import Link from "next/link"; import { Router, useRouter } from "next/router"; import React, { useContext, useEffect, useMemo, useRef, useState } from "react"; -import { - SearchCubeResultOrder, - useOrganizationsQuery, - useThemesQuery, -} from "@/graphql/query-hooks"; -import { useLocale } from "@/locales/use-locale"; +import { SearchCubeResultOrder } from "@/graphql/query-hooks"; import { BrowseParams } from "@/pages/browse"; -import { useDataSourceStore } from "@/stores/data-source"; import useEvent from "@/utils/use-event"; import { getFiltersFromParams } from "./filters"; @@ -114,10 +108,7 @@ const useQueryParamsState = ( }; export const useBrowseState = () => { - const { dataSource } = useDataSourceStore(); - const locale = useLocale(); const inputRef = useRef(null); - const [browseParams, setParams] = useQueryParamsState( {}, { @@ -125,7 +116,6 @@ export const useBrowseState = () => { serialize: buildURLFromBrowseState, } ); - const { search, type, @@ -135,29 +125,9 @@ export const useBrowseState = () => { dataset: paramDataset, } = browseParams; - const [{ data: themeData }] = useThemesQuery({ - variables: { - sourceType: dataSource.type, - sourceUrl: dataSource.url, - locale, - }, - pause: !!paramDataset, - }); - const [{ data: orgData }] = useOrganizationsQuery({ - variables: { - sourceType: dataSource.type, - sourceUrl: dataSource.url, - locale, - }, - pause: !!paramDataset, - }); - // Support /browse?dataset= and legacy /browse/dataset/ const dataset = type === "dataset" ? iri : paramDataset; - const filters = getFiltersFromParams(browseParams, { - themes: themeData?.themes, - organizations: orgData?.organizations, - }); + const filters = getFiltersFromParams(browseParams); const setSearch = useEvent((v: string) => setParams({ search: v })); const setIncludeDrafts = useEvent((v: boolean) => diff --git a/app/browser/dataset-browse.spec.tsx b/app/browser/dataset-browse.spec.tsx index bdd3548a6..842fb0f6e 100644 --- a/app/browser/dataset-browse.spec.tsx +++ b/app/browser/dataset-browse.spec.tsx @@ -1,30 +1,14 @@ -import { DataCubeOrganization, DataCubeTheme } from "@/graphql/query-hooks"; import { BrowseParams } from "@/pages/browse"; import { getFiltersFromParams } from "./filters"; -const ctx = { - themes: [ - { - iri: "https://fake-iri-theme", - __typename: "DataCubeTheme", - }, - ] as DataCubeTheme[], - organizations: [ - { - iri: "https://fake-iri-organization", - __typename: "DataCubeOrganization", - }, - ] as DataCubeOrganization[], -}; - describe("getFiltersFromParams", () => { it("should work only for organization", () => { const params = { type: "organization", iri: "https://fake-iri-organization", } as BrowseParams; - const filters = getFiltersFromParams(params, ctx); + const filters = getFiltersFromParams(params); expect(filters).toEqual([ { __typename: "DataCubeOrganization", @@ -40,7 +24,7 @@ describe("getFiltersFromParams", () => { subtype: "organization", subiri: "https://fake-iri-organization", } as BrowseParams; - const filters = getFiltersFromParams(params, ctx); + const filters = getFiltersFromParams(params); expect(filters).toEqual([ { iri: "https://fake-iri-theme", __typename: "DataCubeTheme" }, { @@ -57,7 +41,7 @@ describe("getFiltersFromParams", () => { subtype: "theme", subiri: "https://fake-iri-theme", } as BrowseParams; - const filters = getFiltersFromParams(params, ctx); + const filters = getFiltersFromParams(params); expect(filters).toEqual([ { iri: "https://fake-iri-organization", @@ -72,7 +56,7 @@ describe("getFiltersFromParams", () => { type: "dataset", iri: "https://fake-iri-dataset", } as BrowseParams; - const filters = getFiltersFromParams(params, ctx); + const filters = getFiltersFromParams(params); expect(filters).toEqual([]); }); }); diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index 6d25f1f4b..1c22bd969 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -39,8 +39,6 @@ import { DataCubeTheme, SearchCubeResultOrder, SearchCubesQuery, - useOrganizationsQuery, - useThemesQuery, } from "@/graphql/query-hooks"; import { DataCubePublicationStatus, @@ -49,8 +47,6 @@ import { import SvgIcCategories from "@/icons/components/IcCategories"; import SvgIcClose from "@/icons/components/IcClose"; import SvgIcOrganisations from "@/icons/components/IcOrganisations"; -import { useLocale } from "@/locales/use-locale"; -import { useDataSourceStore } from "@/stores/data-source"; import isAttrEqual from "@/utils/is-attr-equal"; import useEvent from "@/utils/use-event"; @@ -525,6 +521,7 @@ const NavSection = ({ ); }, [counts, items]); const { isOpen, open, close } = useDisclosure(); + return (
@@ -545,7 +542,7 @@ const NavSection = ({ return ( { - const { dataSource } = useDataSourceStore(); - const locale = useLocale(); - const { filters, dataset } = useBrowseContext(); - const [{ data: allThemes }] = useThemesQuery({ - variables: { - sourceType: dataSource.type, - sourceUrl: dataSource.url, - locale, - }, - pause: !!dataset, - }); - const [{ data: allOrgs }] = useOrganizationsQuery({ - variables: { - sourceType: dataSource.type, - sourceUrl: dataSource.url, - locale, - }, - pause: !!dataset, - }); - +export const SearchFilters = ({ + cubes, + themes, + orgs, +}: { + cubes: SearchCubeResult[]; + themes: DataCubeTheme[]; + orgs: DataCubeOrganization[]; +}) => { + const { filters } = useBrowseContext(); const counts = useMemo(() => { const result: Record = {}; @@ -639,14 +625,7 @@ export const SearchFilters = ({ cubes }: { cubes: SearchCubeResult[] }) => { isAttrEqual("__typename", "DataCubeOrganization") ); - const [allThemesAlpha, allOrgsAlpha] = useMemo(() => { - return [ - allThemes ? sortBy(allThemes.themes, (x) => x?.label) : null, - allOrgs ? sortBy(allOrgs.organizations, (x) => x?.label) : null, - ]; - }, [allThemes, allOrgs]); - - const displayedThemes = allThemesAlpha?.filter((theme) => { + const displayedThemes = themes.filter((theme) => { if (!theme.label) { return false; } @@ -655,23 +634,23 @@ export const SearchFilters = ({ cubes }: { cubes: SearchCubeResult[] }) => { return false; } - if (themeFilter && themeFilter !== theme) { + if (themeFilter && themeFilter.iri !== theme.iri) { return false; } return true; }); - const displayedOrgs = allOrgsAlpha?.filter((org) => { + const displayedOrgs = orgs.filter((org) => { if (!org.label) { return false; } - if (!counts[org.iri] && orgFilter !== org) { + if (!counts[org.iri] && orgFilter?.iri !== org.iri) { return false; } - if (orgFilter && orgFilter !== org) { + if (orgFilter && orgFilter.iri !== org.iri) { return false; } @@ -721,7 +700,7 @@ export const SearchFilters = ({ cubes }: { cubes: SearchCubeResult[] }) => { icon={} label={Organizations} extra={ - orgFilter && filters.includes(orgFilter) ? ( + orgFilter && filters.map((d) => d.iri).includes(orgFilter.iri) ? ( { +export const getFiltersFromParams = (params: BrowseParams) => { const filters: BrowseFilter[] = []; const { type, subtype, iri, subiri, topic } = params; for (const [t, i] of [ @@ -28,17 +17,13 @@ export const getFiltersFromParams = ( [subtype, subiri], ]) { if (t && i && (t === "theme" || t === "organization")) { - const container = context[ - t === "theme" ? "themes" : "organizations" - ] as BrowseFilter[]; - const obj = container?.find((f) => i === f.iri); - if (obj) { - filters.push(obj); - } else { - break; - } + filters.push({ + __typename: t === "theme" ? "DataCubeTheme" : "DataCubeOrganization", + iri: i, + }); } } + if (topic) { filters.push({ __typename: "DataCubeAbout", diff --git a/app/browser/select-dataset-step.tsx b/app/browser/select-dataset-step.tsx index 85e18ae6d..b486ee01e 100644 --- a/app/browser/select-dataset-step.tsx +++ b/app/browser/select-dataset-step.tsx @@ -2,6 +2,8 @@ import { t, Trans } from "@lingui/macro"; import { Box, Button, Theme, Typography } from "@mui/material"; import { makeStyles } from "@mui/styles"; import { AnimatePresence } from "framer-motion"; +import sortBy from "lodash/sortBy"; +import uniqBy from "lodash/uniqBy"; import Head from "next/head"; import NextLink from "next/link"; import { Router, useRouter } from "next/router"; @@ -32,7 +34,12 @@ import { PanelLeftWrapper, PanelMiddleWrapper, } from "@/configurator/components/layout"; -import { useSearchCubesQuery } from "@/graphql/query-hooks"; +import { truthy } from "@/domain/types"; +import { + DataCubeOrganization, + DataCubeTheme, + useSearchCubesQuery, +} from "@/graphql/query-hooks"; import { Icon } from "@/icons"; import { useConfiguratorState, useLocale } from "@/src"; @@ -206,6 +213,32 @@ const SelectDatasetStepContent = () => { }; }, [data, filters]); + const themes: DataCubeTheme[] = React.useMemo(() => { + return sortBy( + uniqBy( + cubes + .flatMap((d) => d.cube.themes) + .map((d) => ({ ...d, __typename: "DataCubeTheme" })), + (d) => d.iri + ), + (d) => d.label + ); + }, [cubes]); + + const orgs: DataCubeOrganization[] = React.useMemo(() => { + return sortBy( + uniqBy( + cubes + .map((d) => d.cube.creator) + .filter((d) => d?.iri) + .filter(truthy) + .map((d) => ({ ...d, __typename: "DataCubeOrganization" })), + (d) => d.iri + ), + (d) => d.label + ); + }, [cubes]); + if (configState.state !== "SELECTING_DATASET") { return null; } @@ -280,7 +313,7 @@ const SelectDatasetStepContent = () => { ) : ( - + )} @@ -329,7 +362,16 @@ const SelectDatasetStepContent = () => { className={classes.filters} variant="h1" > - {queryFilters.map((d) => d.label).join(", ")} + {queryFilters + .map((d) => { + const searchList = + d.type === "DataCubeTheme" ? themes : orgs; + const item = ( + searchList as { iri: string; label?: string }[] + ).find(({ iri }) => iri === d.value); + return (item ?? d).label; + }) + .join(", ")} )} diff --git a/app/graphql/queries/data-cubes.graphql b/app/graphql/queries/data-cubes.graphql index c8dadf889..ccd4b9411 100644 --- a/app/graphql/queries/data-cubes.graphql +++ b/app/graphql/queries/data-cubes.graphql @@ -388,45 +388,6 @@ query PossibleFilters( } } -query Themes($sourceType: String!, $sourceUrl: String!, $locale: String!) { - themes(sourceType: $sourceType, sourceUrl: $sourceUrl, locale: $locale) { - iri - label - } -} - -query Organizations( - $sourceType: String! - $sourceUrl: String! - $locale: String! -) { - organizations( - sourceType: $sourceType - sourceUrl: $sourceUrl - locale: $locale - ) { - iri - label - } -} - -query Subthemes( - $sourceType: String! - $sourceUrl: String! - $locale: String! - $parentIri: String! -) { - subthemes( - sourceType: $sourceType - sourceUrl: $sourceUrl - locale: $locale - parentIri: $parentIri - ) { - label - iri - } -} - fragment hierarchyValueFields on HierarchyValue { value dimensionIri diff --git a/app/graphql/query-hooks.ts b/app/graphql/query-hooks.ts index 12c90b434..f2e65e15f 100644 --- a/app/graphql/query-hooks.ts +++ b/app/graphql/query-hooks.ts @@ -368,9 +368,6 @@ export type Query = { dataCubeByIri?: Maybe; possibleFilters: Array; searchCubes: Array; - themes: Array; - subthemes: Array; - organizations: Array; }; @@ -405,28 +402,6 @@ export type QuerySearchCubesArgs = { }; -export type QueryThemesArgs = { - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; -}; - - -export type QuerySubthemesArgs = { - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; - parentIri: Scalars['String']; -}; - - -export type QueryOrganizationsArgs = { - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; -}; - - export type RelatedDimension = { __typename: 'RelatedDimension'; type: Scalars['String']; @@ -1065,34 +1040,6 @@ export type PossibleFiltersQueryVariables = Exact<{ export type PossibleFiltersQuery = { __typename: 'Query', possibleFilters: Array<{ __typename: 'ObservationFilter', iri: string, type: string, value?: Maybe }> }; -export type ThemesQueryVariables = Exact<{ - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; -}>; - - -export type ThemesQuery = { __typename: 'Query', themes: Array<{ __typename: 'DataCubeTheme', iri: string, label?: Maybe }> }; - -export type OrganizationsQueryVariables = Exact<{ - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; -}>; - - -export type OrganizationsQuery = { __typename: 'Query', organizations: Array<{ __typename: 'DataCubeOrganization', iri: string, label?: Maybe }> }; - -export type SubthemesQueryVariables = Exact<{ - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; - parentIri: Scalars['String']; -}>; - - -export type SubthemesQuery = { __typename: 'Query', subthemes: Array<{ __typename: 'DataCubeTheme', label?: Maybe, iri: string }> }; - export type HierarchyValueFieldsFragment = { __typename: 'HierarchyValue', value: string, dimensionIri: string, depth: number, label: string, alternateName?: Maybe, hasValue?: Maybe, position?: Maybe, identifier?: Maybe }; export type DimensionHierarchyQueryVariables = Exact<{ @@ -1513,47 +1460,6 @@ export const PossibleFiltersDocument = gql` export function usePossibleFiltersQuery(options: Omit, 'query'> = {}) { return Urql.useQuery({ query: PossibleFiltersDocument, ...options }); }; -export const ThemesDocument = gql` - query Themes($sourceType: String!, $sourceUrl: String!, $locale: String!) { - themes(sourceType: $sourceType, sourceUrl: $sourceUrl, locale: $locale) { - iri - label - } -} - `; - -export function useThemesQuery(options: Omit, 'query'> = {}) { - return Urql.useQuery({ query: ThemesDocument, ...options }); -}; -export const OrganizationsDocument = gql` - query Organizations($sourceType: String!, $sourceUrl: String!, $locale: String!) { - organizations(sourceType: $sourceType, sourceUrl: $sourceUrl, locale: $locale) { - iri - label - } -} - `; - -export function useOrganizationsQuery(options: Omit, 'query'> = {}) { - return Urql.useQuery({ query: OrganizationsDocument, ...options }); -}; -export const SubthemesDocument = gql` - query Subthemes($sourceType: String!, $sourceUrl: String!, $locale: String!, $parentIri: String!) { - subthemes( - sourceType: $sourceType - sourceUrl: $sourceUrl - locale: $locale - parentIri: $parentIri - ) { - label - iri - } -} - `; - -export function useSubthemesQuery(options: Omit, 'query'> = {}) { - return Urql.useQuery({ query: SubthemesDocument, ...options }); -}; export const DimensionHierarchyDocument = gql` query DimensionHierarchy($sourceType: String!, $sourceUrl: String!, $locale: String!, $cubeIri: String!, $dimensionIri: String!) { dataCubeByIri( diff --git a/app/graphql/resolver-types.ts b/app/graphql/resolver-types.ts index 27d5afc85..25dfe7f0e 100644 --- a/app/graphql/resolver-types.ts +++ b/app/graphql/resolver-types.ts @@ -369,9 +369,6 @@ export type Query = { dataCubeByIri?: Maybe; possibleFilters: Array; searchCubes: Array; - themes: Array; - subthemes: Array; - organizations: Array; }; @@ -406,28 +403,6 @@ export type QuerySearchCubesArgs = { }; -export type QueryThemesArgs = { - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; -}; - - -export type QuerySubthemesArgs = { - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; - parentIri: Scalars['String']; -}; - - -export type QueryOrganizationsArgs = { - sourceType: Scalars['String']; - sourceUrl: Scalars['String']; - locale: Scalars['String']; -}; - - export type RelatedDimension = { __typename?: 'RelatedDimension'; type: Scalars['String']; @@ -930,9 +905,6 @@ export type QueryResolvers, ParentType, ContextType, RequireFields>; possibleFilters?: Resolver, ParentType, ContextType, RequireFields>; searchCubes?: Resolver, ParentType, ContextType, RequireFields>; - themes?: Resolver, ParentType, ContextType, RequireFields>; - subthemes?: Resolver, ParentType, ContextType, RequireFields>; - organizations?: Resolver, ParentType, ContextType, RequireFields>; }>; export interface RawObservationScalarConfig extends GraphQLScalarTypeConfig { diff --git a/app/graphql/resolvers/index.ts b/app/graphql/resolvers/index.ts index 179345e39..e580022c7 100644 --- a/app/graphql/resolvers/index.ts +++ b/app/graphql/resolvers/index.ts @@ -31,18 +31,6 @@ export const Query: QueryResolvers = { const source = getSource(args.sourceType); return await source.possibleFilters(parent, args, context, info); }, - themes: async (parent, args, context, info) => { - const source = getSource(args.sourceType); - return await source.themes(parent, args, context, info); - }, - subthemes: async (parent, args, context, info) => { - const source = getSource(args.sourceType); - return await source.subthemes(parent, args, context, info); - }, - organizations: async (parent, args, context, info) => { - const source = getSource(args.sourceType); - return await source.organizations(parent, args, context, info); - }, }; const DataCube: DataCubeResolvers = { diff --git a/app/graphql/resolvers/rdf.ts b/app/graphql/resolvers/rdf.ts index 53b3b73a2..709fb76dc 100644 --- a/app/graphql/resolvers/rdf.ts +++ b/app/graphql/resolvers/rdf.ts @@ -5,7 +5,6 @@ import { LRUCache } from "typescript-lru-cache"; import { Filters } from "@/configurator"; import { DimensionValue } from "@/domain/data"; -import { truthy } from "@/domain/types"; import { Loaders } from "@/graphql/context"; import { DataCubeResolvers, @@ -20,11 +19,6 @@ import { getCubeObservations, getResolvedCube, } from "@/rdf/queries"; -import { - loadOrganizations, - loadSubthemes, - loadThemes, -} from "@/rdf/query-cube-metadata"; import { unversionObservation } from "@/rdf/query-dimension-values"; import { queryHierarchy } from "@/rdf/query-hierarchies"; import { SearchResult, searchCubes as _searchCubes } from "@/rdf/query-search"; @@ -135,34 +129,6 @@ export const possibleFilters: NonNullable = return []; }; -export const themes: NonNullable = async ( - _, - { locale }, - { setup }, - info -) => { - const { sparqlClient } = await setup(info); - return (await loadThemes({ locale, sparqlClient })).filter(truthy); -}; - -export const subthemes: NonNullable = async ( - _, - { locale, parentIri }, - { setup }, - info -) => { - const { sparqlClient } = await setup(info); - return (await loadSubthemes({ locale, parentIri, sparqlClient })).filter( - truthy - ); -}; - -export const organizations: NonNullable = - async (_, { locale }, { setup }, info) => { - const { sparqlClient } = await setup(info); - return (await loadOrganizations({ locale, sparqlClient })).filter(truthy); - }; - export const dataCubeDimensions: NonNullable = async ({ cube, locale }, { componentIris }, { setup }, info) => { const { sparqlClient, cache } = await setup(info); diff --git a/app/graphql/resolvers/sql.ts b/app/graphql/resolvers/sql.ts index 2b9c34e1b..bd22e2fab 100644 --- a/app/graphql/resolvers/sql.ts +++ b/app/graphql/resolvers/sql.ts @@ -163,19 +163,6 @@ export const possibleFilters: NonNullable = return []; }; -export const themes: NonNullable = async () => { - return []; -}; - -export const subthemes: NonNullable = async () => { - return []; -}; - -export const organizations: NonNullable = - async () => { - return []; - }; - export const dataCubeDimensions: NonNullable = async ({ cube }) => { // FIXME: type of cube should be different for RDF and SQL. diff --git a/app/graphql/schema.graphql b/app/graphql/schema.graphql index 04cdc66e3..55def3c19 100644 --- a/app/graphql/schema.graphql +++ b/app/graphql/schema.graphql @@ -382,20 +382,4 @@ type Query { includeDrafts: Boolean filters: [SearchCubeFilter!] ): [SearchCubeResult!]! - themes( - sourceType: String! - sourceUrl: String! - locale: String! - ): [DataCubeTheme!]! - subthemes( - sourceType: String! - sourceUrl: String! - locale: String! - parentIri: String! - ): [DataCubeTheme!]! - organizations( - sourceType: String! - sourceUrl: String! - locale: String! - ): [DataCubeOrganization!]! } From 0497a97de887dd5084d8e0c2d7e9d3969c5fc0e0 Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Tue, 31 Oct 2023 14:17:29 +0100 Subject: [PATCH 47/48] fix: Exclude topic when constructing remove URL (search filters nav item) --- app/browser/dataset-browse.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/browser/dataset-browse.tsx b/app/browser/dataset-browse.tsx index 1c22bd969..ebd535f57 100644 --- a/app/browser/dataset-browse.tsx +++ b/app/browser/dataset-browse.tsx @@ -351,7 +351,9 @@ const NavItem = ({ Boolean ) ); - const newFilters = filters.filter((d) => d.iri !== next.iri); + const newFilters = filters.filter( + (f) => f.__typename !== "DataCubeAbout" && f.iri !== next.iri + ); return `/browse/${newFilters .map(encodeFilter) From edb4e919739c15dc87416a13bc8a8a3c1e52d5ea Mon Sep 17 00:00:00 2001 From: Bartosz Prusinowski Date: Tue, 31 Oct 2023 14:28:06 +0100 Subject: [PATCH 48/48] perf: Drop HAVING in favor of direct filtering --- app/rdf/query-search.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/app/rdf/query-search.ts b/app/rdf/query-search.ts index 114af57b0..f20fd440a 100644 --- a/app/rdf/query-search.ts +++ b/app/rdf/query-search.ts @@ -1,4 +1,4 @@ -import { SELECT } from "@tpluscode/sparql-builder"; +import { SELECT, sparql } from "@tpluscode/sparql-builder"; import { descending, group } from "d3"; import { Literal, NamedNode } from "rdf-js"; import ParsingClient from "sparql-http-client/ParsingClient"; @@ -121,7 +121,7 @@ export const searchCubes = async ({ ?.filter((x) => x.type === "DataCubeOrganization") .map((v) => v.value) ?? []; - let scoresQuery = SELECT` + const scoresQuery = SELECT` ?iri ?title ?status ?datePublished ?description ?publisher ?creatorIri ?creatorLabel (GROUP_CONCAT(DISTINCT ?themeIri; SEPARATOR="${GROUP_SEPARATOR}") AS ?themeIris) (GROUP_CONCAT(DISTINCT ?themeLabel; SEPARATOR="${GROUP_SEPARATOR}") AS ?themeLabels) (GROUP_CONCAT(DISTINCT ?subthemeIri; SEPARATOR="${GROUP_SEPARATOR}") AS ?subthemeIris) (GROUP_CONCAT(DISTINCT ?subthemeLabel; SEPARATOR="${GROUP_SEPARATOR}") AS ?subthemeLabels) @@ -166,6 +166,14 @@ export const searchCubes = async ({ })} } } + ${ + themeValues.length + ? sparql` + VALUES ?theme { ${themeValues.map((d) => `<${d}>`).join(" ")} } + ?iri ${ns.dcat.theme} ?theme . + ` + : "" + } # Add more subtheme termsets here when they are available ${ @@ -241,12 +249,6 @@ export const searchCubes = async ({ .THEN.BY`?description`.THEN.BY`?publisher`.THEN.BY`?creatorIri`.THEN .BY`?creatorLabel`.prologue`${pragmas}`; - if (themeValues.length) { - scoresQuery = scoresQuery.HAVING`${themeValues - .map((iri) => `CONTAINS(LCASE(?themeIris), LCASE("${iri}"))`) - .join(" || ")}` as any; - } - const scoreResults = await scoresQuery.execute(sparqlClient.query, { operation: "postUrlencoded", });