From 183cfa964da2c167fb9f8edf016d1bd7ecb8d80a Mon Sep 17 00:00:00 2001 From: "A. Tate Barber" Date: Wed, 20 Jan 2021 15:46:17 -0600 Subject: [PATCH] Add gRPC reflection and file descriptor set support (#1086) * Add gRPC reflection support This change adds a new feature to the gRPC handler that allows graphql-mesh to automatically create a schema by querying the reflection endpoints on a gRPC service that exposes its reflection service. This feature is enabled by adding the grpc-reflection-js package. Additionally, this change replaces dependencies from @grpc/grpc-js with grpc as that is the package that is used with generated proto code. The grpc package works in conjunction with the grpc-reflection-js package. * Revert back to @grpc/grpc-js This change reverts back to using @grpc/grpc-js instead of grpc since the grpc-reflection-js package no longer has conflicting implementation. See https://github.com/redhoyasa/grpc-reflection-js/pull/3 * Update grpc-reflection-js to 0.0.7 * Bump @grpc/grpc-js to 1.1.8 This change syncs @grpc/grpc-js versions for compatibility with grpc-reflection-js. * Add support for grpc file descriptor sets This change adds a new grpc handler configuration property to specify either a binary-encoded or JSON file descriptor set. * Ensure gRPC reflection service roots are not added with relative path This change fixes an issue where the gRPC reflection feature was not correctly adding gathered gRPC service root objects. * Update grpc reflection and descriptor set logic This change updates the grpc handler feature supporting reflection and file descriptor sets using the latest proto-loader api. * Filter out reflection service definitions from grpc handler * add changeset * Fix build Co-authored-by: Arda TANRIKULU --- .changeset/hot-olives-drop.md | 29 ++++++ packages/handlers/grpc/package.json | 1 + packages/handlers/grpc/src/index.ts | 105 +++++++++++++++++---- packages/handlers/grpc/yaml-config.graphql | 10 +- packages/types/src/config-schema.json | 17 +++- packages/types/src/config.ts | 10 +- yarn.lock | 42 ++++++++- 7 files changed, 193 insertions(+), 21 deletions(-) create mode 100644 .changeset/hot-olives-drop.md diff --git a/.changeset/hot-olives-drop.md b/.changeset/hot-olives-drop.md new file mode 100644 index 0000000000000..eb5d5b4498c50 --- /dev/null +++ b/.changeset/hot-olives-drop.md @@ -0,0 +1,29 @@ +--- +'@graphql-mesh/grpc': minor +'@graphql-mesh/types': minor +--- + +feat(grpc): add reflection and file descriptor set support + +This change adds two new features to the gRPC handler. + +- Reflection support +- File descriptor set support + +Both of these features make it easier for `graphql-mesh` to automatically create a schema for gRPC. + +### `useReflection: boolean` + +This config option enables `graphql-mesh` to generate a schema by querying the gRPC reflection endpoints. This feature is enabled by the [`grpc-reflection-js`](https://github.com/redhoyasa/grpc-reflection-js) package. + +### `descriptorSetFilePath: object | string` + +This config option enabled `graphql-mesh` to generate a schema by importing either a binary-encoded file descriptor set file or a JSON file descriptor set file. This config works just like `protoFilePath` and can be a string or an object containing the file and proto loader options. + +Binary-encoded file descriptor sets can be created by using `protoc` with the `--descriptor_set_out` option. Example: + +```sh +protoc -I . --descriptor_set_out=./my-descriptor-set.bin ./my-rpc.proto +``` + +JSON file descriptor sets can be created using [`protobufjs/protobuf.js`](https://github.com/protobufjs/protobuf.js#using-json-descriptors). \ No newline at end of file diff --git a/packages/handlers/grpc/package.json b/packages/handlers/grpc/package.json index c43de132547e2..80568bb112f19 100644 --- a/packages/handlers/grpc/package.json +++ b/packages/handlers/grpc/package.json @@ -20,6 +20,7 @@ "camel-case": "4.1.2", "graphql-scalars": "1.7.0", "graphql-compose": "7.24.1", + "grpc-reflection-js": "0.0.7", "protobufjs": "6.10.2", "pascal-case": "3.1.2", "lodash": "4.17.20" diff --git a/packages/handlers/grpc/src/index.ts b/packages/handlers/grpc/src/index.ts index a388e6b696016..99380c15b9ac2 100644 --- a/packages/handlers/grpc/src/index.ts +++ b/packages/handlers/grpc/src/index.ts @@ -8,15 +8,23 @@ import { credentials, loadPackageDefinition, } from '@grpc/grpc-js'; -import { load } from '@grpc/proto-loader'; +import { + PackageDefinition, + load, + loadFileDescriptorSetFromBuffer, + loadFileDescriptorSetFromObject, +} from '@grpc/proto-loader'; import { camelCase } from 'camel-case'; +import { promises as fsPromises } from 'fs'; import { SchemaComposer } from 'graphql-compose'; import { GraphQLBigInt, GraphQLByte, GraphQLUnsignedInt } from 'graphql-scalars'; import { get } from 'lodash'; import { pascalCase } from 'pascal-case'; -import { AnyNestedObject, IParseOptions, Root } from 'protobufjs'; +import { AnyNestedObject, IParseOptions, Message, Root, RootConstructor } from 'protobufjs'; import { Readable } from 'stream'; import { promisify } from 'util'; +import grpcReflection from 'grpc-reflection-js'; +import { IFileDescriptorSet } from 'protobufjs/ext/descriptor'; import { ClientMethod, @@ -28,6 +36,8 @@ import { getTypeName, } from './utils'; +const { readFile } = fsPromises || {}; + interface LoadOptions extends IParseOptions { includeDirs?: string[]; } @@ -37,6 +47,8 @@ interface GrpcResponseStream> extends Readable cancel(): void; } +type DecodedDescriptorSet = Message & IFileDescriptorSet; + export default class GrpcHandler implements MeshHandler { private config: YamlConfig.GrpcHandler; private cache: KeyValueCache; @@ -84,21 +96,68 @@ export default class GrpcHandler implements MeshHandler { }); const root = new Root(); - let fileName = this.config.protoFilePath; - let options: LoadOptions = {}; - if (typeof this.config.protoFilePath === 'object' && this.config.protoFilePath.file) { - fileName = this.config.protoFilePath.file; - options = this.config.protoFilePath.load; - if (options.includeDirs) { - if (!Array.isArray(options.includeDirs)) { - return Promise.reject(new Error('The includeDirs option must be an array')); + let packageDefinition: PackageDefinition; + if (this.config.useReflection) { + const grpcReflectionServer = this.config.endpoint; + const reflectionClient = new grpcReflection.Client(grpcReflectionServer, creds as any); + const services = (await reflectionClient.listServices()) as string[]; + const serviceRoots = await Promise.all( + services + .filter(s => s && s !== 'grpc.reflection.v1alpha.ServerReflection') + .map((service: string) => reflectionClient.fileContainingSymbol(service)) + ); + serviceRoots.forEach((serviceRoot: Root) => { + if (serviceRoot.nested) { + for (const namespace in serviceRoot.nested) { + if (Object.prototype.hasOwnProperty.call(serviceRoot.nested, namespace)) { + root.add(serviceRoot.nested[namespace]); + } + } } - addIncludePathResolver(root, options.includeDirs); + }); + root.resolveAll(); + const descriptorSet = root.toDescriptor('proto3'); + packageDefinition = loadFileDescriptorSetFromObject(descriptorSet.toJSON()); + } else if (this.config.descriptorSetFilePath) { + // We have to use an ol' fashioned require here :( + // Needed for descriptor.FileDescriptorSet + const descriptor = require('protobufjs/ext/descriptor'); + + let fileName = this.config.descriptorSetFilePath; + let options: LoadOptions = {}; + if (typeof this.config.descriptorSetFilePath === 'object' && this.config.descriptorSetFilePath.file) { + fileName = this.config.descriptorSetFilePath.file; + options = this.config.descriptorSetFilePath.load; + } + const descriptorSetBuffer = await readFile(fileName as string); + let decodedDescriptorSet: DecodedDescriptorSet; + try { + const descriptorSetJSON = JSON.parse(descriptorSetBuffer.toString()); + decodedDescriptorSet = descriptor.FileDescriptorSet.fromObject(descriptorSetJSON) as DecodedDescriptorSet; + packageDefinition = await loadFileDescriptorSetFromObject(descriptorSetJSON, options); + } catch (e) { + decodedDescriptorSet = descriptor.FileDescriptorSet.decode(descriptorSetBuffer) as DecodedDescriptorSet; + packageDefinition = await loadFileDescriptorSetFromBuffer(descriptorSetBuffer, options); } + const descriptorSetRoot = (Root as RootConstructor).fromDescriptor(decodedDescriptorSet); + root.add(descriptorSetRoot); + } else { + let fileName = this.config.protoFilePath; + let options: LoadOptions = {}; + if (typeof this.config.protoFilePath === 'object' && this.config.protoFilePath.file) { + fileName = this.config.protoFilePath.file; + options = this.config.protoFilePath.load; + if (options.includeDirs) { + if (!Array.isArray(options.includeDirs)) { + return Promise.reject(new Error('The includeDirs option must be an array')); + } + addIncludePathResolver(root, options.includeDirs); + } + } + const protoDefinition = await root.load(fileName as string, options); + protoDefinition.resolveAll(); + packageDefinition = await load(fileName as string, options); } - const protoDefinition = await root.load(fileName as string, options); - protoDefinition.resolveAll(); - const packageDefinition = await load(fileName as string, options); const grpcObject = loadPackageDefinition(packageDefinition); const visit = async (nested: AnyNestedObject, name: string, currentPath: string) => { @@ -157,7 +216,18 @@ export default class GrpcHandler implements MeshHandler { if (name !== this.config.serviceName) { rootFieldName = camelCase(name + '_' + rootFieldName); } - if (currentPath !== this.config.packageName) { + if (!this.config.serviceName && this.config.useReflection) { + rootFieldName = camelCase(currentPath.split('.').join('_') + '_' + rootFieldName); + responseType = camelCase(currentPath.split('.').join('_') + '_' + responseType.replace(currentPath, '')); + requestType = camelCase(currentPath.split('.').join('_') + '_' + requestType.replace(currentPath, '')); + } else if (this.config.descriptorSetFilePath && currentPath !== this.config.packageName) { + const reflectionPath = currentPath.replace(this.config.serviceName || '', ''); + rootFieldName = camelCase(currentPath.split('.').join('_') + '_' + rootFieldName); + responseType = camelCase( + currentPath.split('.').join('_') + '_' + responseType.replace(reflectionPath, '') + ); + requestType = camelCase(currentPath.split('.').join('_') + '_' + requestType.replace(reflectionPath, '')); + } else if (currentPath !== this.config.packageName) { rootFieldName = camelCase(currentPath.split('.').join('_') + '_' + rootFieldName); responseType = camelCase(currentPath.split('.').join('_') + '_' + responseType); requestType = camelCase(currentPath.split('.').join('_') + '_' + requestType); @@ -188,7 +258,10 @@ export default class GrpcHandler implements MeshHandler { } else { const clientMethod = promisify(client[methodName].bind(client) as ClientMethod); const identifier = methodName.toLowerCase(); - const rootTC = (identifier.startsWith('get') || identifier.startsWith('list')) ? schemaComposer.Query : schemaComposer.Mutation; + const rootTC = + identifier.startsWith('get') || identifier.startsWith('list') + ? schemaComposer.Query + : schemaComposer.Mutation; rootTC.addFields({ [rootFieldName]: { ...fieldConfig, diff --git a/packages/handlers/grpc/yaml-config.graphql b/packages/handlers/grpc/yaml-config.graphql index 664fef8e059f4..8829e1dccf789 100644 --- a/packages/handlers/grpc/yaml-config.graphql +++ b/packages/handlers/grpc/yaml-config.graphql @@ -13,7 +13,11 @@ type GrpcHandler @md { """ gRPC Proto file that contains your protobuf schema """ - protoFilePath: ProtoFilePathOrString! + protoFilePath: ProtoFilePathOrString + """ + Use a binary-encoded or JSON file descriptor set file + """ + descriptorSetFilePath: ProtoFilePathOrString """ Your base service name Used for naming only @@ -41,6 +45,10 @@ type GrpcHandler @md { MetaData """ metaData: JSON + """ + Use gRPC reflection to automatically gather the connection + """ + useReflection: Boolean } type LoadOptions { diff --git a/packages/types/src/config-schema.json b/packages/types/src/config-schema.json index f577bf6fce2d1..11445de8a7855 100644 --- a/packages/types/src/config-schema.json +++ b/packages/types/src/config-schema.json @@ -581,6 +581,17 @@ } ] }, + "descriptorSetFilePath": { + "description": "Use a binary-encoded or JSON file descriptor set file (Any of: ProtoFilePath, String)", + "anyOf": [ + { + "$ref": "#/definitions/ProtoFilePath" + }, + { + "type": "string" + } + ] + }, "serviceName": { "type": "string", "description": "Your base service name\nUsed for naming only" @@ -605,9 +616,13 @@ "type": "object", "properties": {}, "description": "MetaData" + }, + "useReflection": { + "type": "boolean", + "description": "Use gRPC reflection to automatically gather the connection" } }, - "required": ["endpoint", "protoFilePath"] + "required": ["endpoint"] }, "LoadOptions": { "additionalProperties": false, diff --git a/packages/types/src/config.ts b/packages/types/src/config.ts index fcd9e15b77f0c..38f47cf1004fa 100644 --- a/packages/types/src/config.ts +++ b/packages/types/src/config.ts @@ -193,7 +193,11 @@ export interface GrpcHandler { /** * gRPC Proto file that contains your protobuf schema (Any of: ProtoFilePath, String) */ - protoFilePath: ProtoFilePath | string; + protoFilePath?: ProtoFilePath | string; + /** + * Use a binary-encoded or JSON file descriptor set file (Any of: ProtoFilePath, String) + */ + descriptorSetFilePath?: ProtoFilePath | string; /** * Your base service name * Used for naming only @@ -220,6 +224,10 @@ export interface GrpcHandler { metaData?: { [k: string]: any; }; + /** + * Use gRPC reflection to automatically gather the connection + */ + useReflection?: boolean; } export interface ProtoFilePath { file: string; diff --git a/yarn.lock b/yarn.lock index 63e7d5de77879..50bfab0b53838 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2747,6 +2747,15 @@ google-auth-library "^6.1.1" semver "^6.2.0" +"@grpc/grpc-js@^1.1.7": + version "1.2.3" + resolved "https://registry.yarnpkg.com/@grpc/grpc-js/-/grpc-js-1.2.3.tgz#35dcbca3cb7415ef5ac6e73d44080ebcb44a4be5" + integrity sha512-hMjS4/TiGFtvMxjmM3mgXCw6VIGeI0EWTNzdcV6R+qqCh33dLDcK1wVceAABXKZ+Fia1nETU49RBesOiukQjGA== + dependencies: + "@types/node" "^12.12.47" + google-auth-library "^6.1.1" + semver "^6.2.0" + "@grpc/proto-loader@0.5.5": version "0.5.5" resolved "https://registry.yarnpkg.com/@grpc/proto-loader/-/proto-loader-0.5.5.tgz#6725e7a1827bdf8e92e29fbf4e9ef0203c0906a9" @@ -3716,6 +3725,11 @@ "@types/minimatch" "*" "@types/node" "*" +"@types/google-protobuf@^3.7.2": + version "3.7.4" + resolved "https://registry.yarnpkg.com/@types/google-protobuf/-/google-protobuf-3.7.4.tgz#1621c50ceaf5aefa699851da8e0ea606a2943a39" + integrity sha512-6PjMFKl13cgB4kRdYtvyjKl8VVa0PXS2IdVxHhQ8GEKbxBkyJtSbaIeK1eZGjDKN7dvUh4vkOvU9FMwYNv4GQQ== + "@types/graceful-fs@^4.1.2": version "4.1.4" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.4.tgz#4ff9f641a7c6d1a3508ff88bc3141b152772e753" @@ -3879,7 +3893,14 @@ dependencies: localforage "*" -"@types/lodash@4.14.168": +"@types/lodash.set@^4.3.6": + version "4.3.6" + resolved "https://registry.yarnpkg.com/@types/lodash.set/-/lodash.set-4.3.6.tgz#33e635c2323f855359225df6a5c8c6f1f1908264" + integrity sha512-ZeGDDlnRYTvS31Laij0RsSaguIUSBTYIlJFKL3vm3T2OAZAQj2YpSvVWJc0WiG4jqg9fGX6PAPGvDqBcHfSgFg== + dependencies: + "@types/lodash" "*" + +"@types/lodash@*", "@types/lodash@4.14.168": version "4.14.168" resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.168.tgz#fe24632e79b7ade3f132891afff86caa5e5ce008" integrity sha512-oVfRvqHV/V6D1yifJbVRU3TMp8OT6o6BG+U9MkwuJ3U8/CsDHvalRpsxBqivn71ztOFZBTfJMvETbqHiaNSj7Q== @@ -10435,7 +10456,7 @@ google-p12-pem@^3.0.3: dependencies: node-forge "^0.10.0" -google-protobuf@3.14.0: +google-protobuf@3.14.0, google-protobuf@^3.12.2: version "3.14.0" resolved "https://registry.yarnpkg.com/google-protobuf/-/google-protobuf-3.14.0.tgz#20373d22046e63831a5110e11a84f713cc43651e" integrity sha512-bwa8dBuMpOxg7COyqkW6muQuvNnWgVN8TX/epDRGW5m0jcrmq2QJyCyiV8ZE2/6LaIIqJtiv9bYokFhfpy/o6w== @@ -10835,6 +10856,18 @@ growly@^1.3.0: resolved "https://registry.yarnpkg.com/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081" integrity sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE= +grpc-reflection-js@0.0.7: + version "0.0.7" + resolved "https://registry.yarnpkg.com/grpc-reflection-js/-/grpc-reflection-js-0.0.7.tgz#193d8ef2ffd5c5338cc16ad6edc3e673f9dc6b9f" + integrity sha512-x1t2uv+Sn77VORJpCJIK4vrEBXC91pmw4qGPcILU+6NDT03VHH9pmZWl6ZoVESZ64hZd4qJRjNObHwolUo3MwA== + dependencies: + "@grpc/grpc-js" "^1.1.7" + "@types/google-protobuf" "^3.7.2" + "@types/lodash.set" "^4.3.6" + google-protobuf "^3.12.2" + lodash.set "^4.3.2" + protobufjs "^6.9.0" + gtoken@^5.0.4: version "5.1.0" resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.1.0.tgz#4ba8d2fc9a8459098f76e7e8fd7beaa39fda9fe4" @@ -13587,6 +13620,11 @@ lodash.reject@^4.4.0: resolved "https://registry.yarnpkg.com/lodash.reject/-/lodash.reject-4.6.0.tgz#80d6492dc1470864bbf583533b651f42a9f52415" integrity sha1-gNZJLcFHCGS79YNTO2UfQqn1JBU= +lodash.set@^4.3.2: + version "4.3.2" + resolved "https://registry.yarnpkg.com/lodash.set/-/lodash.set-4.3.2.tgz#d8757b1da807dde24816b0d6a84bea1a76230b23" + integrity sha1-2HV7HagH3eJIFrDWqEvqGnYjCyM= + lodash.some@^4.4.0: version "4.6.0" resolved "https://registry.yarnpkg.com/lodash.some/-/lodash.some-4.6.0.tgz#1bb9f314ef6b8baded13b549169b2a945eb68e4d"