Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf: Optimize dataset preview performance #1299

Merged
merged 24 commits into from
Feb 13, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
956520e
refactor: Simplify
bprusinowski Dec 12, 2023
406c3b3
feat: Add initial optimized DataCubePreview
bprusinowski Feb 6, 2024
c0e44c2
refactor: Simplify dataset preview query
bprusinowski Feb 7, 2024
877e3ee
fix: Missing prefix
bprusinowski Feb 7, 2024
9a44e90
chore: Remove unused
bprusinowski Feb 7, 2024
ded8bd6
fix: Remove duplicated query statement
bprusinowski Feb 7, 2024
3be52f3
refactor: Use shorter variable names
bprusinowski Feb 7, 2024
b011e04
perf: Filter out unnecessary predicates
bprusinowski Feb 7, 2024
68a9515
refactor: Names
bprusinowski Feb 7, 2024
ead7f27
refactor: Share more parsing logic
bprusinowski Feb 7, 2024
ecc92e0
fix: Unit-related data retrievals
bprusinowski Feb 7, 2024
9467c6d
fix: Correctly retrieve observation values
bprusinowski Feb 7, 2024
be59a75
perf: Improve dataset preview performance by nesting select queries
bprusinowski Feb 7, 2024
8207acc
refactor: Remove unnecessary part of the query
bprusinowski Feb 12, 2024
b04b2d6
perf: Do not fetch rdf:type for observations
bprusinowski Feb 12, 2024
d6e07ac
feat: Add a way to pass additional fallbacks
bprusinowski Feb 12, 2024
b95d5fd
perf: Do not query both IRIs and literals
bprusinowski Feb 12, 2024
f172240
feat: Query dimension value positions
bprusinowski Feb 12, 2024
851b576
fix: Retrieve data type is a correct way
bprusinowski Feb 12, 2024
c0a28cb
refactor: Remove ShortQuad
bprusinowski Feb 12, 2024
a1ef79f
refactor: Clean up
bprusinowski Feb 12, 2024
e0e6d21
test: Dataset preview query
bprusinowski Feb 12, 2024
c056e03
perf: Do not fetch data type separately
bprusinowski Feb 12, 2024
11d1c1d
perf: Reduce number of .finds
bprusinowski Feb 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 5 additions & 17 deletions app/browse/datatable.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -80,24 +80,16 @@ export const PreviewTable = ({
const [sortDirection, setSortDirection] = useState<"asc" | "desc">();
const formatters = useDimensionFormatters(headers);
const sortedObservations = useMemo(() => {
if (sortBy === undefined) {
if (!sortBy) {
return observations;
}

const compare = sortDirection === "asc" ? ascending : descending;
const valuesIndex = uniqueMapBy(sortBy.values, (x) => x.label);
const valuesIndex = uniqueMapBy(sortBy.values, (d) => d.label);
const convert =
isNumericalMeasure(sortBy) || sortBy.isNumerical
? (d: string) => +d
: (d: string) => {
const value = valuesIndex.get(d);

if (value?.position) {
return value.position;
}

return d;
};
? (value: string) => +value
: (value: string) => valuesIndex.get(value)?.position ?? value;

return [...observations].sort((a, b) =>
compare(
Expand All @@ -111,11 +103,7 @@ export const PreviewTable = ({

// Tooltip contained inside the table so as not to overflow when table is scrolled
const tooltipProps = useMemo(
() => ({
PopperProps: {
container: tooltipContainerRef.current,
},
}),
() => ({ PopperProps: { container: tooltipContainerRef.current } }),
[]
);

Expand Down
84 changes: 32 additions & 52 deletions app/browser/dataset-preview.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ import * as React from "react";

import { DataSetPreviewTable } from "@/browse/datatable";
import { useFootnotesStyles } from "@/components/chart-footnotes";
import { DataDownloadMenu, RunSparqlQuery } from "@/components/data-download";
import { DataDownloadMenu } from "@/components/data-download";
import Flex from "@/components/flex";
import { HintRed, Loading, LoadingDataError } from "@/components/hint";
import { DataSource } from "@/config-types";
import { sourceToLabel } from "@/domain/datasource";
import { useDataCubesComponentsQuery } from "@/graphql/hooks";
import { useDataCubePreviewQuery } from "@/graphql/query-hooks";
import {
useDataCubeMetadataQuery,
useDataCubePreviewQuery,
} from "@/graphql/query-hooks";
import { DataCubePublicationStatus } from "@/graphql/resolver-types";
import { useLocale } from "@/locales/use-locale";

Expand Down Expand Up @@ -93,55 +95,38 @@ export const DataSetPreview = ({
}) => {
const footnotesClasses = useFootnotesStyles({ useMarginTop: false });
const locale = useLocale();
const cubeFilters = [{ iri: dataSetIri }];
const variables = {
sourceType: dataSource.type,
sourceUrl: dataSource.url,
locale,
cubeFilter: { iri: dataSetIri },
};
const [{ data: metadata, fetching: fetchingMetadata, error: metadataError }] =
useDataCubeMetadataQuery({ variables });
const [
{ data: previewData, fetching: fetchingPreview, error: previewError },
] = useDataCubePreviewQuery({
variables: {
iri: dataSetIri,
sourceType: dataSource.type,
sourceUrl: dataSource.url,
locale,
},
});
const [
{
data: componentsData,
fetching: fetchingComponents,
error: componentsError,
},
] = useDataCubesComponentsQuery({
variables: {
sourceType: dataSource.type,
sourceUrl: dataSource.url,
locale,
cubeFilters,
},
});
] = useDataCubePreviewQuery({ variables });
const classes = useStyles({
descriptionPresent: !!previewData?.dataCubeByIri?.description,
descriptionPresent: !!metadata?.dataCubeMetadata.description,
});

React.useEffect(() => {
window.scrollTo({ top: 0 });
}, []);

if (fetchingPreview || fetchingComponents) {
if (fetchingMetadata || fetchingPreview) {
return (
<Flex className={classes.loadingWrapper}>
<Loading />
</Flex>
);
} else if (
previewData?.dataCubeByIri &&
componentsData?.dataCubesComponents
) {
const { dataCubeByIri } = previewData;
const { dataCubesComponents } = componentsData;
} else if (metadata?.dataCubeMetadata && previewData?.dataCubePreview) {
const { dataCubeMetadata } = metadata;
const { dataCubePreview } = previewData;

return (
<Flex className={classes.root}>
{dataCubeByIri.publicationStatus ===
{dataCubeMetadata.publicationStatus ===
DataCubePublicationStatus.Draft && (
<Box sx={{ mb: 4 }}>
<HintRed iconName="datasetError" iconSize={64}>
Expand All @@ -156,15 +141,15 @@ export const DataSetPreview = ({
<Flex className={classes.header}>
<Head>
<title key="title">
{dataCubeByIri.title} - visualize.admin.ch
{dataCubeMetadata.title} - visualize.admin.ch
</title>
</Head>
<Typography className={classes.title} component="div" variant="h1">
{dataCubeByIri.title}
{dataCubeMetadata.title}
</Typography>
<Link
href={`/create/new?cube=${
dataCubeByIri.iri
dataCubeMetadata.iri
}&dataSource=${sourceToLabel(dataSource)}`}
passHref
legacyBehavior
Expand All @@ -177,36 +162,31 @@ export const DataSetPreview = ({
</Link>
</Flex>
<Paper className={classes.paper} elevation={5}>
{dataCubeByIri.description && (
{dataCubeMetadata.description && (
<Typography
className={classes.description}
component="div"
variant="body2"
>
{dataCubeByIri.description}
{dataCubeMetadata.description}
</Typography>
)}

<Flex className={classes.tableWrapper}>
<DataSetPreviewTable
title={dataCubeByIri.title}
dimensions={dataCubesComponents.dimensions}
measures={dataCubesComponents.measures}
observations={dataCubeByIri.observations.data}
title={dataCubeMetadata.title}
dimensions={dataCubePreview.dimensions}
measures={dataCubePreview.measures}
observations={dataCubePreview.observations}
/>
</Flex>
<Flex className={classes.footnotesWrapper}>
<Flex className={footnotesClasses.actions}>
<DataDownloadMenu
dataSource={dataSource}
title={dataCubeByIri.title}
filters={cubeFilters}
title={dataCubeMetadata.title}
filters={[variables.cubeFilter]}
/>
{dataCubeByIri.observations.sparqlEditorUrl && (
<RunSparqlQuery
url={dataCubeByIri.observations.sparqlEditorUrl}
/>
)}
</Flex>
<Typography
className={classes.numberOfRows}
Expand All @@ -225,7 +205,7 @@ export const DataSetPreview = ({
return (
<Flex className={classes.loadingWrapper}>
<LoadingDataError
message={previewError?.message ?? componentsError?.message}
message={metadataError?.message ?? previewError?.message}
/>
</Flex>
);
Expand Down
36 changes: 21 additions & 15 deletions app/domain/data.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,21 +40,6 @@ export type HierarchyValue = {
children?: HierarchyValue[];
};

export type Observation = Record<string, ObservationValue>;

export type DataCubeObservations = {
data: Observation[];
sparqlEditorUrl: string;
};

export type DataCubesObservations = {
data: Observation[];
sparqlEditorUrls: {
cubeIri: string;
url: string;
}[];
};

export type DataCubeComponents = {
dimensions: Dimension[];
measures: Measure[];
Expand Down Expand Up @@ -82,6 +67,27 @@ export type DataCubeMetadata = {
workExamples?: string[];
};

export type Observation = Record<string, ObservationValue>;

export type DataCubeObservations = {
data: Observation[];
sparqlEditorUrl: string;
};

export type DataCubesObservations = {
data: Observation[];
sparqlEditorUrls: {
cubeIri: string;
url: string;
}[];
};

export type DataCubePreview = {
dimensions: Dimension[];
measures: Measure[];
observations: Observation[];
};

export type Component = Dimension | Measure;

export type BaseComponent = {
Expand Down
46 changes: 14 additions & 32 deletions app/graphql/queries/data-cubes.graphql
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,20 @@ query DataCubeObservations(
)
}

query DataCubePreview(
$sourceType: String!
$sourceUrl: String!
$locale: String!
$cubeFilter: DataCubePreviewFilter!
) {
dataCubePreview(
sourceType: $sourceType
sourceUrl: $sourceUrl
locale: $locale
cubeFilter: $cubeFilter
)
}

query SearchCubes(
$sourceType: String!
$sourceUrl: String!
Expand All @@ -64,38 +78,6 @@ query SearchCubes(
}
}

query DataCubePreview(
$iri: String!
$sourceType: String!
$sourceUrl: String!
$locale: String!
$latest: Boolean
$disableValuesLoad: Boolean = true
) {
dataCubeByIri(
iri: $iri
sourceType: $sourceType
sourceUrl: $sourceUrl
locale: $locale
latest: $latest
disableValuesLoad: $disableValuesLoad
) {
iri
title
description
publicationStatus
observations(
sourceType: $sourceType
sourceUrl: $sourceUrl
preview: true
limit: 10
) {
data
sparqlEditorUrl
}
}
}

query GeoCoordinatesByDimensionIri(
$dataCubeIri: String!
$dimensionIri: String!
Expand Down
Loading