Add gRPC reflection and file descriptor set support (#1086)

* Add gRPC reflection support This change adds a new feature to the gRPC handler that allows graphql-mesh to automatically create a schema by querying the reflection endpoints on a gRPC service that exposes its reflection service. This feature is enabled by adding the grpc-reflection-js package. Additionally, this change replaces dependencies from @grpc/grpc-js with grpc as that is the package that is used with generated proto code. The grpc package works in conjunction with the grpc-reflection-js package. * Revert back to @grpc/grpc-js This change reverts back to using @grpc/grpc-js instead of grpc since the grpc-reflection-js package no longer has conflicting implementation. See redhoyasa/grpc-reflection-js#3 * Update grpc-reflection-js to 0.0.7 * Bump @grpc/grpc-js to 1.1.8 This change syncs @grpc/grpc-js versions for compatibility with grpc-reflection-js. * Add support for grpc file descriptor sets This change adds a new grpc handler configuration property to specify either a binary-encoded or JSON file descriptor set. * Ensure gRPC reflection service roots are not added with relative path This change fixes an issue where the gRPC reflection feature was not correctly adding gathered gRPC service root objects. * Update grpc reflection and descriptor set logic This change updates the grpc handler feature supporting reflection and file descriptor sets using the latest proto-loader api. * Filter out reflection service definitions from grpc handler * add changeset * Fix build Co-authored-by: Arda TANRIKULU <[email protected]>
ardatan · Jan 20, 2021 · 183cfa9 · 183cfa9
1 parent c767df0
commit 183cfa9
Show file tree

Hide file tree

Showing 7 changed files with 193 additions and 21 deletions.
diff --git a/.changeset/hot-olives-drop.md b/.changeset/hot-olives-drop.md
@@ -0,0 +1,29 @@
+---
+'@graphql-mesh/grpc': minor
+'@graphql-mesh/types': minor
+---
+
+feat(grpc): add reflection and file descriptor set support
+
+This change adds two new features to the gRPC handler.
+
+- Reflection support
+- File descriptor set support
+
+Both of these features make it easier for `graphql-mesh` to automatically create a schema for gRPC.
+
+### `useReflection: boolean`
+
+This config option enables `graphql-mesh` to generate a schema by querying the gRPC reflection endpoints. This feature is enabled by the [`grpc-reflection-js`](https://github.com/redhoyasa/grpc-reflection-js) package.
+
+### `descriptorSetFilePath: object | string`
+
+This config option enabled `graphql-mesh` to generate a schema by importing either a binary-encoded file descriptor set file or a JSON file descriptor set file. This config works just like `protoFilePath` and can be a string or an object containing the file and proto loader options.
+
+Binary-encoded file descriptor sets can be created by using `protoc` with the `--descriptor_set_out` option. Example:
+
+```sh
+protoc -I . --descriptor_set_out=./my-descriptor-set.bin ./my-rpc.proto
+```
+
+JSON file descriptor sets can be created using [`protobufjs/protobuf.js`](https://github.com/protobufjs/protobuf.js#using-json-descriptors).
diff --git a/packages/handlers/grpc/package.json b/packages/handlers/grpc/package.json
@@ -20,6 +20,7 @@
     "camel-case": "4.1.2",
     "graphql-scalars": "1.7.0",
     "graphql-compose": "7.24.1",
+    "grpc-reflection-js": "0.0.7",
     "protobufjs": "6.10.2",
     "pascal-case": "3.1.2",
     "lodash": "4.17.20"

diff --git a/packages/handlers/grpc/src/index.ts b/packages/handlers/grpc/src/index.ts
@@ -8,15 +8,23 @@ import {
   credentials,
   loadPackageDefinition,
 } from '@grpc/grpc-js';
-import { load } from '@grpc/proto-loader';
+import {
+  PackageDefinition,
+  load,
+  loadFileDescriptorSetFromBuffer,
+  loadFileDescriptorSetFromObject,
+} from '@grpc/proto-loader';
 import { camelCase } from 'camel-case';
+import { promises as fsPromises } from 'fs';
 import { SchemaComposer } from 'graphql-compose';
 import { GraphQLBigInt, GraphQLByte, GraphQLUnsignedInt } from 'graphql-scalars';
 import { get } from 'lodash';
 import { pascalCase } from 'pascal-case';
-import { AnyNestedObject, IParseOptions, Root } from 'protobufjs';
+import { AnyNestedObject, IParseOptions, Message, Root, RootConstructor } from 'protobufjs';
 import { Readable } from 'stream';
 import { promisify } from 'util';
+import grpcReflection from 'grpc-reflection-js';
+import { IFileDescriptorSet } from 'protobufjs/ext/descriptor';
 
 import {
   ClientMethod,
@@ -28,6 +36,8 @@ import {
   getTypeName,
 } from './utils';
 
+const { readFile } = fsPromises || {};
+
 interface LoadOptions extends IParseOptions {
   includeDirs?: string[];
 }
@@ -37,6 +47,8 @@ interface GrpcResponseStream<T = ClientReadableStream<unknown>> extends Readable
   cancel(): void;
 }
 
+type DecodedDescriptorSet = Message<IFileDescriptorSet> & IFileDescriptorSet;
+
 export default class GrpcHandler implements MeshHandler {
   private config: YamlConfig.GrpcHandler;
   private cache: KeyValueCache;
@@ -84,21 +96,68 @@ export default class GrpcHandler implements MeshHandler {
     });
 
     const root = new Root();
-    let fileName = this.config.protoFilePath;
-    let options: LoadOptions = {};
-    if (typeof this.config.protoFilePath === 'object' && this.config.protoFilePath.file) {
-      fileName = this.config.protoFilePath.file;
-      options = this.config.protoFilePath.load;
-      if (options.includeDirs) {
-        if (!Array.isArray(options.includeDirs)) {
-          return Promise.reject(new Error('The includeDirs option must be an array'));
+    let packageDefinition: PackageDefinition;
+    if (this.config.useReflection) {
+      const grpcReflectionServer = this.config.endpoint;
+      const reflectionClient = new grpcReflection.Client(grpcReflectionServer, creds as any);
+      const services = (await reflectionClient.listServices()) as string[];
+      const serviceRoots = await Promise.all(
+        services
+          .filter(s => s && s !== 'grpc.reflection.v1alpha.ServerReflection')
+          .map((service: string) => reflectionClient.fileContainingSymbol(service))
+      );
+      serviceRoots.forEach((serviceRoot: Root) => {
+        if (serviceRoot.nested) {
+          for (const namespace in serviceRoot.nested) {
+            if (Object.prototype.hasOwnProperty.call(serviceRoot.nested, namespace)) {
+              root.add(serviceRoot.nested[namespace]);
+            }
+          }
         }
-        addIncludePathResolver(root, options.includeDirs);
+      });
+      root.resolveAll();
+      const descriptorSet = root.toDescriptor('proto3');
+      packageDefinition = loadFileDescriptorSetFromObject(descriptorSet.toJSON());
+    } else if (this.config.descriptorSetFilePath) {
+      // We have to use an ol' fashioned require here :(
+      // Needed for descriptor.FileDescriptorSet
+      const descriptor = require('protobufjs/ext/descriptor');
+
+      let fileName = this.config.descriptorSetFilePath;
+      let options: LoadOptions = {};
+      if (typeof this.config.descriptorSetFilePath === 'object' && this.config.descriptorSetFilePath.file) {
+        fileName = this.config.descriptorSetFilePath.file;
+        options = this.config.descriptorSetFilePath.load;
+      }
+      const descriptorSetBuffer = await readFile(fileName as string);
+      let decodedDescriptorSet: DecodedDescriptorSet;
+      try {
+        const descriptorSetJSON = JSON.parse(descriptorSetBuffer.toString());
+        decodedDescriptorSet = descriptor.FileDescriptorSet.fromObject(descriptorSetJSON) as DecodedDescriptorSet;
+        packageDefinition = await loadFileDescriptorSetFromObject(descriptorSetJSON, options);
+      } catch (e) {
+        decodedDescriptorSet = descriptor.FileDescriptorSet.decode(descriptorSetBuffer) as DecodedDescriptorSet;
+        packageDefinition = await loadFileDescriptorSetFromBuffer(descriptorSetBuffer, options);
       }
+      const descriptorSetRoot = (Root as RootConstructor).fromDescriptor(decodedDescriptorSet);
+      root.add(descriptorSetRoot);
+    } else {
+      let fileName = this.config.protoFilePath;
+      let options: LoadOptions = {};
+      if (typeof this.config.protoFilePath === 'object' && this.config.protoFilePath.file) {
+        fileName = this.config.protoFilePath.file;
+        options = this.config.protoFilePath.load;
+        if (options.includeDirs) {
+          if (!Array.isArray(options.includeDirs)) {
+            return Promise.reject(new Error('The includeDirs option must be an array'));
+          }
+          addIncludePathResolver(root, options.includeDirs);
+        }
+      }
+      const protoDefinition = await root.load(fileName as string, options);
+      protoDefinition.resolveAll();
+      packageDefinition = await load(fileName as string, options);
     }
-    const protoDefinition = await root.load(fileName as string, options);
-    protoDefinition.resolveAll();
-    const packageDefinition = await load(fileName as string, options);
     const grpcObject = loadPackageDefinition(packageDefinition);
 
     const visit = async (nested: AnyNestedObject, name: string, currentPath: string) => {
@@ -157,7 +216,18 @@ export default class GrpcHandler implements MeshHandler {
             if (name !== this.config.serviceName) {
               rootFieldName = camelCase(name + '_' + rootFieldName);
             }
-            if (currentPath !== this.config.packageName) {
+            if (!this.config.serviceName && this.config.useReflection) {
+              rootFieldName = camelCase(currentPath.split('.').join('_') + '_' + rootFieldName);
+              responseType = camelCase(currentPath.split('.').join('_') + '_' + responseType.replace(currentPath, ''));
+              requestType = camelCase(currentPath.split('.').join('_') + '_' + requestType.replace(currentPath, ''));
+            } else if (this.config.descriptorSetFilePath && currentPath !== this.config.packageName) {
+              const reflectionPath = currentPath.replace(this.config.serviceName || '', '');
+              rootFieldName = camelCase(currentPath.split('.').join('_') + '_' + rootFieldName);
+              responseType = camelCase(
+                currentPath.split('.').join('_') + '_' + responseType.replace(reflectionPath, '')
+              );
+              requestType = camelCase(currentPath.split('.').join('_') + '_' + requestType.replace(reflectionPath, ''));
+            } else if (currentPath !== this.config.packageName) {
               rootFieldName = camelCase(currentPath.split('.').join('_') + '_' + rootFieldName);
               responseType = camelCase(currentPath.split('.').join('_') + '_' + responseType);
               requestType = camelCase(currentPath.split('.').join('_') + '_' + requestType);
@@ -188,7 +258,10 @@ export default class GrpcHandler implements MeshHandler {
             } else {
               const clientMethod = promisify<ClientUnaryCall>(client[methodName].bind(client) as ClientMethod);
               const identifier = methodName.toLowerCase();
-              const rootTC = (identifier.startsWith('get') || identifier.startsWith('list')) ? schemaComposer.Query : schemaComposer.Mutation;
+              const rootTC =
+                identifier.startsWith('get') || identifier.startsWith('list')
+                  ? schemaComposer.Query
+                  : schemaComposer.Mutation;
               rootTC.addFields({
                 [rootFieldName]: {
                   ...fieldConfig,

diff --git a/packages/handlers/grpc/yaml-config.graphql b/packages/handlers/grpc/yaml-config.graphql
@@ -13,7 +13,11 @@ type GrpcHandler @md {
   """
   gRPC Proto file that contains your protobuf schema
   """
-  protoFilePath: ProtoFilePathOrString!
+  protoFilePath: ProtoFilePathOrString
+  """
+  Use a binary-encoded or JSON file descriptor set file
+  """
+  descriptorSetFilePath: ProtoFilePathOrString
   """
   Your base service name
   Used for naming only
@@ -41,6 +45,10 @@ type GrpcHandler @md {
   MetaData
   """
   metaData: JSON
+  """
+  Use gRPC reflection to automatically gather the connection
+  """
+  useReflection: Boolean
 }
 
 type LoadOptions {

diff --git a/packages/types/src/config-schema.json b/packages/types/src/config-schema.json
@@ -581,6 +581,17 @@
             }
           ]
         },
+        "descriptorSetFilePath": {
+          "description": "Use a binary-encoded or JSON file descriptor set file (Any of: ProtoFilePath, String)",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/ProtoFilePath"
+            },
+            {
+              "type": "string"
+            }
+          ]
+        },
         "serviceName": {
           "type": "string",
           "description": "Your base service name\nUsed for naming only"
@@ -605,9 +616,13 @@
           "type": "object",
           "properties": {},
           "description": "MetaData"
+        },
+        "useReflection": {
+          "type": "boolean",
+          "description": "Use gRPC reflection to automatically gather the connection"
         }
       },
-      "required": ["endpoint", "protoFilePath"]
+      "required": ["endpoint"]
     },
     "LoadOptions": {
       "additionalProperties": false,

diff --git a/packages/types/src/config.ts b/packages/types/src/config.ts
@@ -193,7 +193,11 @@ export interface GrpcHandler {
   /**
    * gRPC Proto file that contains your protobuf schema (Any of: ProtoFilePath, String)
    */
-  protoFilePath: ProtoFilePath | string;
+  protoFilePath?: ProtoFilePath | string;
+  /**
+   * Use a binary-encoded or JSON file descriptor set file (Any of: ProtoFilePath, String)
+   */
+  descriptorSetFilePath?: ProtoFilePath | string;
   /**
    * Your base service name
    * Used for naming only
@@ -220,6 +224,10 @@ export interface GrpcHandler {
   metaData?: {
     [k: string]: any;
   };
+  /**
+   * Use gRPC reflection to automatically gather the connection
+   */
+  useReflection?: boolean;
 }
 export interface ProtoFilePath {
   file: string;

diff --git a/yarn.lock b/yarn.lock
@@ -2747,6 +2747,15 @@
     google-auth-library "^6.1.1"
     semver "^6.2.0"
 
+"@grpc/grpc-js@^1.1.7":
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.2.3.tgz#35dcbca3cb7415ef5ac6e73d44080ebcb44a4be5"
+  integrity sha512-hMjS4/TiGFtvMxjmM3mgXCw6VIGeI0EWTNzdcV6R+qqCh33dLDcK1wVceAABXKZ+Fia1nETU49RBesOiukQjGA==
+  dependencies:
+    "@types/node" "^12.12.47"
+    google-auth-library "^6.1.1"
+    semver "^6.2.0"
+
 "@grpc/[email protected]":
   version "0.5.5"
   resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.5.5.tgz#6725e7a1827bdf8e92e29fbf4e9ef0203c0906a9"
@@ -3716,6 +3725,11 @@
     "@types/minimatch" "*"
     "@types/node" "*"
 
+"@types/google-protobuf@^3.7.2":
+  version "3.7.4"
+  resolved "https://registry.yarnpkg.com/@types/google-protobuf/-/google-protobuf-3.7.4.tgz#1621c50ceaf5aefa699851da8e0ea606a2943a39"
+  integrity sha512-6PjMFKl13cgB4kRdYtvyjKl8VVa0PXS2IdVxHhQ8GEKbxBkyJtSbaIeK1eZGjDKN7dvUh4vkOvU9FMwYNv4GQQ==
+
 "@types/graceful-fs@^4.1.2":
   version "4.1.4"
   resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.4.tgz#4ff9f641a7c6d1a3508ff88bc3141b152772e753"
@@ -3879,7 +3893,14 @@
   dependencies:
     localforage "*"
 
-"@types/[email protected]":
+"@types/lodash.set@^4.3.6":
+  version "4.3.6"
+  resolved "https://registry.yarnpkg.com/@types/lodash.set/-/lodash.set-4.3.6.tgz#33e635c2323f855359225df6a5c8c6f1f1908264"
+  integrity sha512-ZeGDDlnRYTvS31Laij0RsSaguIUSBTYIlJFKL3vm3T2OAZAQj2YpSvVWJc0WiG4jqg9fGX6PAPGvDqBcHfSgFg==
+  dependencies:
+    "@types/lodash" "*"
+
+"@types/lodash@*", "@types/[email protected]":
   version "4.14.168"
   resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.168.tgz#fe24632e79b7ade3f132891afff86caa5e5ce008"
   integrity sha512-oVfRvqHV/V6D1yifJbVRU3TMp8OT6o6BG+U9MkwuJ3U8/CsDHvalRpsxBqivn71ztOFZBTfJMvETbqHiaNSj7Q==
@@ -10435,7 +10456,7 @@ google-p12-pem@^3.0.3:
   dependencies:
     node-forge "^0.10.0"
 
-[email protected]:
+[email protected], google-protobuf@^3.12.2:
   version "3.14.0"
   resolved "https://registry.yarnpkg.com/google-protobuf/-/google-protobuf-3.14.0.tgz#20373d22046e63831a5110e11a84f713cc43651e"
   integrity sha512-bwa8dBuMpOxg7COyqkW6muQuvNnWgVN8TX/epDRGW5m0jcrmq2QJyCyiV8ZE2/6LaIIqJtiv9bYokFhfpy/o6w==
@@ -10835,6 +10856,18 @@ growly@^1.3.0:
   resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081"
   integrity sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=
 
+[email protected]:
+  version "0.0.7"
+  resolved "https://registry.yarnpkg.com/grpc-reflection-js/-/grpc-reflection-js-0.0.7.tgz#193d8ef2ffd5c5338cc16ad6edc3e673f9dc6b9f"
+  integrity sha512-x1t2uv+Sn77VORJpCJIK4vrEBXC91pmw4qGPcILU+6NDT03VHH9pmZWl6ZoVESZ64hZd4qJRjNObHwolUo3MwA==
+  dependencies:
+    "@grpc/grpc-js" "^1.1.7"
+    "@types/google-protobuf" "^3.7.2"
+    "@types/lodash.set" "^4.3.6"
+    google-protobuf "^3.12.2"
+    lodash.set "^4.3.2"
+    protobufjs "^6.9.0"
+
 gtoken@^5.0.4:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.1.0.tgz#4ba8d2fc9a8459098f76e7e8fd7beaa39fda9fe4"
@@ -13587,6 +13620,11 @@ lodash.reject@^4.4.0:
   resolved "https://registry.yarnpkg.com/lodash.reject/-/lodash.reject-4.6.0.tgz#80d6492dc1470864bbf583533b651f42a9f52415"
   integrity sha1-gNZJLcFHCGS79YNTO2UfQqn1JBU=
 
+lodash.set@^4.3.2:
+  version "4.3.2"
+  resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23"
+  integrity sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM=
+
 lodash.some@^4.4.0:
   version "4.6.0"
   resolved "https://registry.yarnpkg.com/lodash.some/-/lodash.some-4.6.0.tgz#1bb9f314ef6b8baded13b549169b2a945eb68e4d"