Add distinct query plugin

packages/node-core/src/indexer/dictionary.service.ts

@@ -27,6 +27,8 @@ export type SpecVersionDictionary = {
 
 const logger = getLogger('dictionary');
 
+const distinctErrorEscaped = `Unknown argument \\"distinct\\"`;
+
 function extractVar(name: string, cond: DictionaryQueryCondition): GqlVar {
   let gqlType: string;
   switch (typeof cond.value) {
@@ -96,7 +98,8 @@ function buildDictQueryFragment(
   startBlock: number,
   queryEndBlock: number,
   conditions: DictionaryQueryCondition[][],
-  batchSize: number
+  batchSize: number,
+  useDistinct: boolean,
 ): [GqlVar[], GqlNode] {
   const [gqlVars, filter] = extractVars(entity, conditions);
 
@@ -120,6 +123,11 @@ function buildDictQueryFragment(
       first: batchSize.toString(),
     },
   };
+
+  if (useDistinct) {
+    node.args.distinct = ['BLOCK_HEIGHT'];
+  }
+
   return [gqlVars, node];
 }
 
@@ -128,6 +136,7 @@ export class DictionaryService implements OnApplicationShutdown {
   protected client: ApolloClient<NormalizedCacheObject>;
   private isShutdown = false;
   private mappedDictionaryQueryEntries: Map<number, DictionaryQueryEntry[]>;
+  private useDistinct = true;
 
   constructor(
     readonly dictionaryEndpoint: string,
@@ -197,6 +206,18 @@ export class DictionaryService implements OnApplicationShutdown {
         batchBlocks,
       };
     } catch (err) {
+      // Check if the error is about distinct argument and disable distinct if so
+      if (JSON.stringify(err).includes(distinctErrorEscaped)) {
+        this.useDistinct = false;
+        logger.warn(`Dictionary doesn't support distinct query.`);
+        // Rerun the qeury now with distinct disabled
+        return this.getDictionary(
+          startBlock,
+          queryEndBlock,
+          batchSize,
+          conditions,
+        );
+      }
       logger.warn(err, `failed to fetch dictionary result`);
       return undefined;
     }
@@ -223,7 +244,7 @@ export class DictionaryService implements OnApplicationShutdown {
       },
     ];
     for (const entity of Object.keys(mapped)) {
-      const [pVars, node] = buildDictQueryFragment(entity, startBlock, queryEndBlock, mapped[entity], batchSize);
+      const [pVars, node] = buildDictQueryFragment(entity, startBlock, queryEndBlock, mapped[entity], batchSize, this.useDistinct);
       nodes.push(node);
       vars.push(...pVars);
     }

packages/query/src/graphql/plugins/PgDistinctPlugin.ts

@@ -0,0 +1,107 @@
+// Copyright 2022 OnFinality Limited authors & contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import {Build, Plugin} from 'graphile-build';
+import {PgClass, QueryBuilder} from 'graphile-build-pg';
+import type {GraphQLEnumType} from 'graphql';
+import * as PgSql from 'pg-sql2';
+
+type Extend = <T1, T2>(base: T1, extra: T2, hint?: string) => T1 & T2;
+
+const getEnumName = (entityName: string): string => {
+  return `${entityName}_distinct_enum`;
+};
+
+export const PgDistinctPlugin: Plugin = (builder) => {
+  // Creates enums for each entity based on their fields
+  builder.hook(
+    'init',
+    (args, build) => {
+      const {
+        graphql: {GraphQLEnumType},
+        newWithHooks,
+        pgIntrospectionResultsByKind,
+      } = build;
+
+      pgIntrospectionResultsByKind.class.forEach((cls: PgClass) => {
+        if (!cls.isSelectable || build.pgOmit(cls, 'order')) return;
+        if (!cls.namespace) return;
+
+        const enumTypeName = getEnumName(cls.name);
+
+        const entityEnumValues: Record<string, {value: number}> = {};
+        cls?.attributes?.forEach((attr, index) => {
+          if (attr.name.indexOf('_') !== 0) {
+            entityEnumValues[attr.name.toUpperCase()] = {value: index};
+          }
+        });
+
+        newWithHooks(
+          GraphQLEnumType,
+          {
+            name: enumTypeName,
+            values: entityEnumValues,
+          },
+          {
+            __origin: `Adding connection "distinct" enum type for ${cls.name}.`,
+            pgIntrospection: cls,
+          },
+          true
+        );
+      });
+
+      return args;
+    },
+    ['AddDistinctEnumsPlugin']
+  );
+
+  // Extends schema and modifies the query
+  builder.hook(
+    'GraphQLObjectType:fields:field:args',
+    (args, build, {addArgDataGenerator, scope: {pgFieldIntrospection}}) => {
+      const {
+        extend,
+        graphql: {GraphQLList},
+        pgSql: sql,
+      } = build as Build & {extend: Extend; pgSql: typeof PgSql};
+
+      const enumTypeName = getEnumName(pgFieldIntrospection?.name);
+      const enumType = build.getTypeByName(enumTypeName) as GraphQLEnumType;
+
+      if (!enumType) {
+        return args;
+      }
+
+      addArgDataGenerator(({distinct}) => ({
+        pgQuery: (queryBuilder: QueryBuilder) => {
+          distinct?.map((field: number) => {
+            const {name} = enumType.getValues()[field];
+            const fieldName = name.toLowerCase();
+            if (!pgFieldIntrospection?.attributes?.map((a) => a.name).includes(fieldName)) {
+              console.warn(`Distinct field ${fieldName} doesn't exist on entity ${pgFieldIntrospection?.name}`);
+
+              return;
+            }
+
+            const id = sql.fragment`${queryBuilder.getTableAlias()}.${sql.identifier(fieldName)}`;
+
+            // Dependent on https://github.com/graphile/graphile-engine/pull/805
+            (queryBuilder as any).distinctOn(id);
+          });
+        },
+      }));
+
+      return extend(
+        args,
+        {
+          distinct: {
+            description: 'Fields to be distinct',
+            defaultValue: null,
+            type: new GraphQLList(enumType),
+          },
+        },
+        'DistinctPlugin'
+      );
+    }
+  );
+};

packages/query/src/graphql/plugins/index.ts

@@ -52,6 +52,7 @@ import {makeAddInflectorsPlugin} from 'graphile-utils';
 import PgAggregationPlugin from './PgAggregationPlugin';
 import {PgBlockHeightPlugin} from './PgBlockHeightPlugin';
 import {PgRowByVirtualIdPlugin} from './PgRowByVirtualIdPlugin';
+import {PgDistinctPlugin} from './PgDistinctPlugin';
 
 /* eslint-enable */
 
@@ -109,6 +110,7 @@ const plugins = [
   PgAggregationPlugin,
   PgBlockHeightPlugin,
   PgRowByVirtualIdPlugin,
+  PgDistinctPlugin,
   makeAddInflectorsPlugin((inflectors) => {
     const {constantCase: oldConstantCase} = inflectors;
     const enumValues = new Set();