From 71061810f6f1a15b71729ae07c6866ceb29395bf Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 29 Aug 2022 15:53:24 +1200 Subject: [PATCH 1/5] Add distinct query plugin --- packages/query/src/graphql/graphql.module.ts | 1 + .../src/graphql/plugins/PgDistinctPlugin.ts | 106 ++++++++++++++++++ packages/query/src/graphql/plugins/index.ts | 2 + 3 files changed, 109 insertions(+) create mode 100644 packages/query/src/graphql/plugins/PgDistinctPlugin.ts diff --git a/packages/query/src/graphql/graphql.module.ts b/packages/query/src/graphql/graphql.module.ts index 4573775853..2bc7ea4949 100644 --- a/packages/query/src/graphql/graphql.module.ts +++ b/packages/query/src/graphql/graphql.module.ts @@ -91,6 +91,7 @@ export class GraphqlModule implements OnModuleInit, OnModuleDestroy { const graphqlSchema = builder.buildSchema(); return graphqlSchema; } catch (e) { + console.log('SCHEMA ERROR', e); await delay(SCHEMA_RETRY_INTERVAL); if (retries === 1) { logger.error(e); diff --git a/packages/query/src/graphql/plugins/PgDistinctPlugin.ts b/packages/query/src/graphql/plugins/PgDistinctPlugin.ts new file mode 100644 index 0000000000..9f82288474 --- /dev/null +++ b/packages/query/src/graphql/plugins/PgDistinctPlugin.ts @@ -0,0 +1,106 @@ +// Copyright 2022 OnFinality Limited authors & contributors +// SPDX-License-Identifier: Apache-2.0 + +import {Build, Plugin} from 'graphile-build'; +import {PgClass, QueryBuilder} from 'graphile-build-pg'; +import type {GraphQLEnumType} from 'graphql'; +import * as PgSql from 'pg-sql2'; + +type Extend = (base: T1, extra: T2, hint?: string) => T1 & T2; + +const getEnumName = (entityName: string): string => { + return `${entityName}_distinct_enum`; +}; + +export const PgDistinctPlugin: Plugin = (builder) => { + // Creates enums for each entity based on their fields + builder.hook( + 'init', + (args, build) => { + const { + graphql: {GraphQLEnumType}, + newWithHooks, + pgIntrospectionResultsByKind, + } = build; + + pgIntrospectionResultsByKind.class.forEach((cls: PgClass) => { + if (!cls.isSelectable || build.pgOmit(cls, 'order')) return; + if (!cls.namespace) return; + + const enumTypeName = getEnumName(cls.name); + + const entityEnumValues: Record = {}; + cls?.attributes?.forEach((attr, index) => { + if (attr.name.indexOf('_') !== 0) { + entityEnumValues[attr.name] = {value: index}; + } + }); + + newWithHooks( + GraphQLEnumType, + { + name: enumTypeName, + values: entityEnumValues, + }, + { + __origin: `Adding connection "distinct" enum type for ${cls.name}.`, + pgIntrospection: cls, + }, + true + ); + }); + + return args; + }, + ['AddDistinctEnumsPlugin'] + ); + + // Extends schema and modifies the query + builder.hook( + 'GraphQLObjectType:fields:field:args', + (args, build, {addArgDataGenerator, scope: {pgFieldIntrospection}}) => { + const { + extend, + graphql: {GraphQLList}, + pgSql: sql, + } = build as Build & {extend: Extend; pgSql: typeof PgSql}; + + const enumTypeName = getEnumName(pgFieldIntrospection?.name); + const enumType = build.getTypeByName(enumTypeName) as GraphQLEnumType; + + if (!enumType) { + return args; + } + + addArgDataGenerator(({distinct}) => ({ + pgQuery: (queryBuilder: QueryBuilder) => { + distinct.map((field: number) => { + const {name: fieldName} = enumType.getValues()[field]; + if (!pgFieldIntrospection?.attributes?.map((a) => a.name).includes(fieldName)) { + console.warn(`Distinct field ${fieldName} doesn't exist on entity ${pgFieldIntrospection?.name}`); + + return; + } + + const id = sql.fragment`${queryBuilder.getTableAlias()}.${sql.identifier(fieldName)}`; + + // Dependent on https://github.com/graphile/graphile-engine/pull/805 + (queryBuilder as any).distinctOn(id); + }); + }, + })); + + return extend( + args, + { + distinct: { + description: 'Fields to be distinct', + defaultValue: null, + type: new GraphQLList(enumType), + }, + }, + 'DistinctPlugin' + ); + } + ); +}; diff --git a/packages/query/src/graphql/plugins/index.ts b/packages/query/src/graphql/plugins/index.ts index 39a3834ed4..56c48ea846 100644 --- a/packages/query/src/graphql/plugins/index.ts +++ b/packages/query/src/graphql/plugins/index.ts @@ -52,6 +52,7 @@ import {makeAddInflectorsPlugin} from 'graphile-utils'; import PgAggregationPlugin from './PgAggregationPlugin'; import {PgBlockHeightPlugin} from './PgBlockHeightPlugin'; import {PgRowByVirtualIdPlugin} from './PgRowByVirtualIdPlugin'; +import {PgDistinctPlugin} from './PgDistinctPlugin'; /* eslint-enable */ @@ -109,6 +110,7 @@ const plugins = [ PgAggregationPlugin, PgBlockHeightPlugin, PgRowByVirtualIdPlugin, + PgDistinctPlugin, makeAddInflectorsPlugin((inflectors) => { const {constantCase: oldConstantCase} = inflectors; const enumValues = new Set(); From a9ce714fd6562933e8d8b90aefc97d2ff5076be1 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Mon, 29 Aug 2022 16:04:23 +1200 Subject: [PATCH 2/5] Clean up log --- packages/query/src/graphql/graphql.module.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/query/src/graphql/graphql.module.ts b/packages/query/src/graphql/graphql.module.ts index 2bc7ea4949..4573775853 100644 --- a/packages/query/src/graphql/graphql.module.ts +++ b/packages/query/src/graphql/graphql.module.ts @@ -91,7 +91,6 @@ export class GraphqlModule implements OnModuleInit, OnModuleDestroy { const graphqlSchema = builder.buildSchema(); return graphqlSchema; } catch (e) { - console.log('SCHEMA ERROR', e); await delay(SCHEMA_RETRY_INTERVAL); if (retries === 1) { logger.error(e); From f6327e9aebd9a6738fced163765265c113cdc117 Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 30 Aug 2022 13:46:19 +1200 Subject: [PATCH 3/5] Fix distinct not being provided to query --- packages/query/src/graphql/plugins/PgDistinctPlugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/query/src/graphql/plugins/PgDistinctPlugin.ts b/packages/query/src/graphql/plugins/PgDistinctPlugin.ts index 9f82288474..3fa792e450 100644 --- a/packages/query/src/graphql/plugins/PgDistinctPlugin.ts +++ b/packages/query/src/graphql/plugins/PgDistinctPlugin.ts @@ -74,7 +74,7 @@ export const PgDistinctPlugin: Plugin = (builder) => { addArgDataGenerator(({distinct}) => ({ pgQuery: (queryBuilder: QueryBuilder) => { - distinct.map((field: number) => { + distinct?.map((field: number) => { const {name: fieldName} = enumType.getValues()[field]; if (!pgFieldIntrospection?.attributes?.map((a) => a.name).includes(fieldName)) { console.warn(`Distinct field ${fieldName} doesn't exist on entity ${pgFieldIntrospection?.name}`); From c151d9274961e91c998cffdb5d402470e642ae3f Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 8 Sep 2022 13:54:25 +1200 Subject: [PATCH 4/5] Uppercase enum to be consistent with other enums --- packages/query/src/graphql/plugins/PgDistinctPlugin.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/query/src/graphql/plugins/PgDistinctPlugin.ts b/packages/query/src/graphql/plugins/PgDistinctPlugin.ts index 3fa792e450..2311b1d28c 100644 --- a/packages/query/src/graphql/plugins/PgDistinctPlugin.ts +++ b/packages/query/src/graphql/plugins/PgDistinctPlugin.ts @@ -32,7 +32,7 @@ export const PgDistinctPlugin: Plugin = (builder) => { const entityEnumValues: Record = {}; cls?.attributes?.forEach((attr, index) => { if (attr.name.indexOf('_') !== 0) { - entityEnumValues[attr.name] = {value: index}; + entityEnumValues[attr.name.toUpperCase()] = {value: index}; } }); @@ -75,7 +75,8 @@ export const PgDistinctPlugin: Plugin = (builder) => { addArgDataGenerator(({distinct}) => ({ pgQuery: (queryBuilder: QueryBuilder) => { distinct?.map((field: number) => { - const {name: fieldName} = enumType.getValues()[field]; + const {name} = enumType.getValues()[field]; + const fieldName = name.toLowerCase(); if (!pgFieldIntrospection?.attributes?.map((a) => a.name).includes(fieldName)) { console.warn(`Distinct field ${fieldName} doesn't exist on entity ${pgFieldIntrospection?.name}`); From 51728427be79fff787a71c2dedd560f12638557f Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Thu, 8 Sep 2022 15:25:27 +1200 Subject: [PATCH 5/5] Update dictionary queries to try distinct argument --- .../src/indexer/dictionary.service.ts | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/packages/node-core/src/indexer/dictionary.service.ts b/packages/node-core/src/indexer/dictionary.service.ts index 818fc0d532..79c5e7e374 100644 --- a/packages/node-core/src/indexer/dictionary.service.ts +++ b/packages/node-core/src/indexer/dictionary.service.ts @@ -27,6 +27,8 @@ export type SpecVersionDictionary = { const logger = getLogger('dictionary'); +const distinctErrorEscaped = `Unknown argument \\"distinct\\"`; + function extractVar(name: string, cond: DictionaryQueryCondition): GqlVar { let gqlType: string; switch (typeof cond.value) { @@ -96,7 +98,8 @@ function buildDictQueryFragment( startBlock: number, queryEndBlock: number, conditions: DictionaryQueryCondition[][], - batchSize: number + batchSize: number, + useDistinct: boolean, ): [GqlVar[], GqlNode] { const [gqlVars, filter] = extractVars(entity, conditions); @@ -120,6 +123,11 @@ function buildDictQueryFragment( first: batchSize.toString(), }, }; + + if (useDistinct) { + node.args.distinct = ['BLOCK_HEIGHT']; + } + return [gqlVars, node]; } @@ -128,6 +136,7 @@ export class DictionaryService implements OnApplicationShutdown { protected client: ApolloClient; private isShutdown = false; private mappedDictionaryQueryEntries: Map; + private useDistinct = true; constructor( readonly dictionaryEndpoint: string, @@ -197,6 +206,18 @@ export class DictionaryService implements OnApplicationShutdown { batchBlocks, }; } catch (err) { + // Check if the error is about distinct argument and disable distinct if so + if (JSON.stringify(err).includes(distinctErrorEscaped)) { + this.useDistinct = false; + logger.warn(`Dictionary doesn't support distinct query.`); + // Rerun the qeury now with distinct disabled + return this.getDictionary( + startBlock, + queryEndBlock, + batchSize, + conditions, + ); + } logger.warn(err, `failed to fetch dictionary result`); return undefined; } @@ -223,7 +244,7 @@ export class DictionaryService implements OnApplicationShutdown { }, ]; for (const entity of Object.keys(mapped)) { - const [pVars, node] = buildDictQueryFragment(entity, startBlock, queryEndBlock, mapped[entity], batchSize); + const [pVars, node] = buildDictQueryFragment(entity, startBlock, queryEndBlock, mapped[entity], batchSize, this.useDistinct); nodes.push(node); vars.push(...pVars); }