diff --git a/packages/rest/package.json b/packages/rest/package.json index db0a4fa15b20..61eedb906435 100644 --- a/packages/rest/package.json +++ b/packages/rest/package.json @@ -38,6 +38,7 @@ "http-errors": "^1.6.3", "js-yaml": "^3.11.0", "lodash": "^4.17.5", + "multiparty": "^4.2.1", "openapi-schema-to-json-schema": "^2.1.0", "openapi3-ts": "^1.0.0", "parseurl": "^1.3.2", @@ -54,6 +55,7 @@ "@types/debug": "0.0.30", "@types/js-yaml": "^3.11.1", "@types/lodash": "^4.14.106", + "@types/multiparty": "0.0.31", "@types/node": "^10.11.2" }, "files": [ diff --git a/packages/rest/src/keys.ts b/packages/rest/src/keys.ts index e5389b339e1b..688fbc3bb710 100644 --- a/packages/rest/src/keys.ts +++ b/packages/rest/src/keys.ts @@ -25,6 +25,7 @@ import { ParseParams, Reject, Send, + RequestBodyParserOptions, } from './types'; import {HttpProtocol} from '@loopback/http-server'; @@ -84,6 +85,10 @@ export namespace RestBindings { 'rest.errorWriterOptions', ); + export const REQUEST_BODY_PARSER_OPTIONS = BindingKey.create< + RequestBodyParserOptions + >('rest.requestBodyParserOptions'); + /** * Binding key for setting and injecting an OpenAPI spec */ diff --git a/packages/rest/src/parser.ts b/packages/rest/src/parser.ts index 9cd608b89378..cdb038d472e4 100644 --- a/packages/rest/src/parser.ts +++ b/packages/rest/src/parser.ts @@ -19,8 +19,14 @@ import {promisify} from 'util'; import {coerceParameter} from './coercion/coerce-parameter'; import {RestHttpErrors} from './index'; import {ResolvedRoute} from './router/routing-table'; -import {OperationArgs, PathParameterValues, Request} from './types'; +import { + OperationArgs, + PathParameterValues, + Request, + RequestBodyParserOptions, +} from './types'; import {validateRequestBody} from './validation/request-body.validator'; +import {Form} from 'multiparty'; type HttpError = HttpErrors.HttpError; @@ -29,12 +35,21 @@ const debug = debugModule('loopback:rest:parser'); export const QUERY_NOT_PARSED = {}; Object.freeze(QUERY_NOT_PARSED); -// tslint:disable-next-line:no-any -type MaybeBody = any | undefined; +// tslint:disable:no-any +type RequestBody = { + value: any | undefined; + coercionRequired?: boolean; +}; + +const parseJsonBody: ( + req: IncomingMessage, + options: {}, +) => Promise = promisify(require('body/json')); -const parseJsonBody: (req: IncomingMessage) => Promise = promisify( - require('body/json'), -); +const parseFormBody: ( + req: IncomingMessage, + options: {}, +) => Promise = promisify(require('body/form')); /** * Get the content-type header value from the request @@ -61,11 +76,12 @@ function getContentType(req: Request): string | undefined { export async function parseOperationArgs( request: Request, route: ResolvedRoute, + options: RequestBodyParserOptions = {}, ): Promise { debug('Parsing operation arguments for route %s', route.describe()); const operationSpec = route.spec; const pathParams = route.pathParams; - const body = await loadRequestBodyIfNeeded(operationSpec, request); + const body = await loadRequestBodyIfNeeded(operationSpec, request, options); return buildOperationArguments( operationSpec, request, @@ -75,32 +91,105 @@ export async function parseOperationArgs( ); } +async function parseMultiParty(request: IncomingMessage, options: any) { + return new Promise((resolve, reject) => { + const form = new Form(options); + + form.parse(request, function (error: Error, fields: any, files: any) { + if (error) { + return reject(error); + } + + const json: any = {}; + for (const key of Object.keys(fields)) { + const value = fields[key]; + + if (value instanceof Array) { + json[key] = value[0]; + } else { + json[key] = value; + } + } + resolve({...json, files: files}); + }); + }); +} + async function loadRequestBodyIfNeeded( operationSpec: OperationObject, request: Request, -): Promise { - if (!operationSpec.requestBody) return Promise.resolve(); + options: RequestBodyParserOptions = {}, +): Promise { + if (!operationSpec.requestBody) return Promise.resolve({value: undefined}); + + debug('Request body parser options: %j', options); const contentType = getContentType(request); debug('Loading request body with content type %j', contentType); + + if ( + contentType && + contentType.startsWith('multipart/form-data') + ) { + const body = await parseMultiParty(request, options).catch( + (err: HttpError) => { + debug('Cannot parse request body %j', err); + if (!err.statusCode || err.statusCode >= 500) { + err.statusCode = 400; + } + throw err; + }, + ); + // form parser returns an object with prototype + return { + value: Object.assign({}, body), + coercionRequired: true, + }; + } + + if ( + contentType && + contentType.startsWith('application/x-www-form-urlencoded') + ) { + const body = await parseFormBody(request, options).catch( + (err: HttpError) => { + debug('Cannot parse request body %j', err); + if (!err.statusCode || err.statusCode >= 500) { + err.statusCode = 400; + } + throw err; + }, + ); + // form parser returns an object with prototype + return { + value: Object.assign({}, body), + coercionRequired: true, + }; + } + if (contentType && !/json/.test(contentType)) { throw new HttpErrors.UnsupportedMediaType( `Content-type ${contentType} is not supported.`, ); } - return await parseJsonBody(request).catch((err: HttpError) => { - debug('Cannot parse request body %j', err); - err.statusCode = 400; - throw err; - }); + const jsonBody = await parseJsonBody(request, options).catch( + (err: HttpError) => { + debug('Cannot parse request body %j', err); + if (!err.statusCode || err.statusCode >= 500) { + err.statusCode = 400; + } + throw err; + }, + ); + return {value: jsonBody}; } function buildOperationArguments( operationSpec: OperationObject, request: Request, pathParams: PathParameterValues, - body: MaybeBody, + body: RequestBody, globalSchemas: SchemasObject, ): OperationArgs { let requestBodyIndex: number = -1; @@ -129,9 +218,11 @@ function buildOperationArguments( } debug('Validating request body - value %j', body); - validateRequestBody(body, operationSpec.requestBody, globalSchemas); + validateRequestBody(body.value, operationSpec.requestBody, globalSchemas, { + coerceTypes: body.coercionRequired, + }); - if (requestBodyIndex > -1) paramArgs.splice(requestBodyIndex, 0, body); + if (requestBodyIndex > -1) paramArgs.splice(requestBodyIndex, 0, body.value); return paramArgs; } diff --git a/packages/rest/src/providers/parse-params.provider.ts b/packages/rest/src/providers/parse-params.provider.ts index f37443cc875a..ad53a5c9006d 100644 --- a/packages/rest/src/providers/parse-params.provider.ts +++ b/packages/rest/src/providers/parse-params.provider.ts @@ -3,18 +3,26 @@ // This file is licensed under the MIT License. // License text available at https://opensource.org/licenses/MIT -import {Provider, BoundValue} from '@loopback/context'; +import {Provider, inject} from '@loopback/context'; import {parseOperationArgs} from '../parser'; +import {RestBindings} from '../keys'; +import {ResolvedRoute} from '../router'; +import {Request, ParseParams, RequestBodyParserOptions} from '../types'; /** * Provides the function for parsing args in requests at runtime. * * @export * @class ParseParamsProvider - * @implements {Provider} - * @returns {BoundValue} The handler function that will parse request args. + * @implements {Provider} + * @returns {ParseParams} The handler function that will parse request args. */ -export class ParseParamsProvider implements Provider { +export class ParseParamsProvider implements Provider { + constructor( + @inject(RestBindings.REQUEST_BODY_PARSER_OPTIONS, {optional: true}) + private options?: RequestBodyParserOptions, + ) {} value() { - return parseOperationArgs; + return (request: Request, route: ResolvedRoute) => + parseOperationArgs(request, route, this.options); } } diff --git a/packages/rest/src/types.ts b/packages/rest/src/types.ts index cebcbaabc649..bd57ab7151c1 100644 --- a/packages/rest/src/types.ts +++ b/packages/rest/src/types.ts @@ -77,6 +77,16 @@ export type LogError = ( ) => void; // tslint:disable:no-any + +/** + * Options for request body parsing + */ +export type RequestBodyParserOptions = { + limit?: number; + encoding?: string; + [property: string]: any; +}; + export type PathParameterValues = {[key: string]: any}; export type OperationArgs = any[]; @@ -90,3 +100,33 @@ export type OperationRetval = any; export type GetFromContext = (key: string) => Promise; export type BindElement = (key: string) => Binding; + +export interface File { + /** + * same as name - the field name for this file + */ + fieldName: string; + /** + * the filename that the user reports for the file + */ + originalFilename: string; + /** + * the absolute path of the uploaded file on disk + */ + path: string; + /** + * the HTTP headers that were sent along with this file + */ + headers: any; + /** + * size of the file in bytes + */ + size: number; +} + +export interface FileArray { + /* + * the file arrays + */ + file?: File[]; +} diff --git a/packages/rest/src/validation/request-body.validator.ts b/packages/rest/src/validation/request-body.validator.ts index 3fa057237b7f..a937bb76786c 100644 --- a/packages/rest/src/validation/request-body.validator.ts +++ b/packages/rest/src/validation/request-body.validator.ts @@ -32,6 +32,7 @@ export function validateRequestBody( body: any, requestBodySpec: RequestBodyObject | undefined, globalSchemas?: SchemasObject, + options?: AJV.Options, ) { if (!requestBodySpec) return; @@ -50,7 +51,7 @@ export function validateRequestBody( debug('Request body schema: %j', util.inspect(schema, {depth: null})); if (!schema) return; - validateValueAgainstSchema(body, schema, globalSchemas); + validateValueAgainstSchema(body, schema, globalSchemas, options); } /** @@ -94,13 +95,14 @@ function validateValueAgainstSchema( body: any, schema: SchemaObject, globalSchemas?: SchemasObject, + options?: AJV.Options, ) { let validate; if (compiledSchemaCache.has(schema)) { validate = compiledSchemaCache.get(schema); } else { - validate = createValidator(schema, globalSchemas); + validate = createValidator(schema, globalSchemas, options); compiledSchemaCache.set(schema, validate); } @@ -128,6 +130,7 @@ function validateValueAgainstSchema( function createValidator( schema: SchemaObject, globalSchemas?: SchemasObject, + options?: AJV.Options, ): Function { const jsonSchema = convertToJsonSchema(schema); @@ -136,9 +139,15 @@ function createValidator( schemas: globalSchemas, }; - const ajv = new AJV({ - allErrors: true, - }); + const ajv = new AJV( + Object.assign( + {}, + { + allErrors: true, + }, + options, + ), + ); return ajv.compile(schemaWithRef); } diff --git a/packages/rest/test/integration/http-handler.integration.ts b/packages/rest/test/integration/http-handler.integration.ts index 7051367c0486..d99c5e79606a 100644 --- a/packages/rest/test/integration/http-handler.integration.ts +++ b/packages/rest/test/integration/http-handler.integration.ts @@ -7,7 +7,6 @@ import { HttpHandler, DefaultSequence, writeResultToResponse, - parseOperationArgs, RestBindings, FindRouteProvider, InvokeMethodProvider, @@ -21,6 +20,7 @@ import {ParameterObject, RequestBodyObject} from '@loopback/openapi-v3-types'; import {anOpenApiSpec, anOperationSpec} from '@loopback/openapi-spec-builder'; import {createUnexpectedHttpErrorLogger} from '../helpers'; import * as express from 'express'; +import {ParseParamsProvider} from '../../src'; const SequenceActions = RestBindings.SequenceActions; @@ -242,19 +242,11 @@ describe('HttpHandler', () => { .expect(200, {key: 'value'}); }); - it('rejects url-encoded request body', () => { - logErrorsExcept(415); + it('allows url-encoded request body', () => { return client .post('/show-body') .send('key=value') - .expect(415, { - error: { - message: - 'Content-type application/x-www-form-urlencoded is not supported.', - name: 'UnsupportedMediaTypeError', - statusCode: 415, - }, - }); + .expect(200, {key: 'value'}); }); it('returns 400 for malformed JSON body', () => { @@ -272,7 +264,83 @@ describe('HttpHandler', () => { }); }); + it('rejects unsupported request body', () => { + logErrorsExcept(415); + return client + .post('/show-body') + .set('content-type', 'application/xml') + .send('value') + .expect(415, { + error: { + message: 'Content-type application/xml is not supported.', + name: 'UnsupportedMediaTypeError', + statusCode: 415, + }, + }); + }); + + let bodyParamControllerInvoked = false; + it('rejects over-limit request form body', () => { + logErrorsExcept(413); + return client + .post('/show-body') + .set('content-type', 'application/x-www-form-urlencoded') + .send('key=' + givenLargeRequest()) + .expect(413, { + error: { + message: 'request entity too large', + name: 'Error', + statusCode: 413, + }, + }) + .catch(catchEpipeError) + .then(() => expect(bodyParamControllerInvoked).be.false()); + }); + + it('rejects over-limit request json body', () => { + logErrorsExcept(413); + return client + .post('/show-body') + .set('content-type', 'application/json') + .send({key: givenLargeRequest()}) + .expect(413, { + error: { + message: 'request entity too large', + name: 'Error', + statusCode: 413, + }, + }) + .catch(catchEpipeError) + .then(() => expect(bodyParamControllerInvoked).be.false()); + }); + + it('allows customization of request body parser options', () => { + const body = {key: givenLargeRequest()}; + rootContext + .bind(RestBindings.REQUEST_BODY_PARSER_OPTIONS) + .to({limit: 4 * 1024 * 1024}); // Set limit to 4MB + return client + .post('/show-body') + .set('content-type', 'application/json') + .send(body) + .expect(200, body); + }); + + function catchEpipeError(err: HttpErrors.HttpError) { + // The server side can close the socket before the client + // side can send out all data. For example, `response.end()` + // is called before all request data has been processed due + // to size limit + if (err && err.code !== 'EPIPE') throw err; + } + + function givenLargeRequest() { + const data = Buffer.alloc(2 * 1024 * 1024, 'A', 'utf-8'); + return data.toString(); + } + function givenBodyParamController() { + bodyParamControllerInvoked = false; const spec = anOpenApiSpec() .withOperation('post', '/show-body', { 'x-operation-name': 'showBody', @@ -302,6 +370,7 @@ describe('HttpHandler', () => { class RouteParamController { async showBody(data: Object): Promise { + bodyParamControllerInvoked = true; return data; } } @@ -472,7 +541,9 @@ describe('HttpHandler', () => { function givenHandler() { rootContext = new Context(); rootContext.bind(SequenceActions.FIND_ROUTE).toProvider(FindRouteProvider); - rootContext.bind(SequenceActions.PARSE_PARAMS).to(parseOperationArgs); + rootContext + .bind(SequenceActions.PARSE_PARAMS) + .toProvider(ParseParamsProvider); rootContext .bind(SequenceActions.INVOKE_METHOD) .toProvider(InvokeMethodProvider); diff --git a/packages/rest/test/unit/parser.unit.ts b/packages/rest/test/unit/parser.unit.ts index 2faafc691312..59b616a4a825 100644 --- a/packages/rest/test/unit/parser.unit.ts +++ b/packages/rest/test/unit/parser.unit.ts @@ -21,9 +21,115 @@ import { Request, RestHttpErrors, Route, + File, } from '../..'; describe('operationArgsParser', () => { + it('parses body parameter for multipart form data with simple types', async () => { + const req = givenRequest({ + url: '/', + headers: { + 'Content-Type': 'multipart/form-data; boundary=--------------------------961406998938770617032718', + }, + payload: '----------------------------961406998938770617032718\r\n' + + 'Content-Disposition: form-data; name="key1"\r\n\r\n' + 'value\r\n' + + '----------------------------961406998938770617032718\r\n' + + 'Content-Disposition: form-data; name="key2"\r\n\r\n' + '1\r\n' + + '----------------------------961406998938770617032718\r\n' + + 'Content-Disposition: form-data; name="key3"\r\n\r\n' + 'true\r\n' + + '----------------------------961406998938770617032718--\r\n', + }); + + const spec = givenOperationWithRequestBody({ + description: 'data', + content: { + 'multipart/form-data': { + schema: { + type: 'object', + properties: { + key1: {type: 'string'}, + key2: {type: 'number'}, + key3: {type: 'boolean'}, + files: {type: 'object'}, + }, + }, + }, + }, + }); + const route = givenResolvedRoute(spec); + + const args = await parseOperationArgs(req, route); + + expect(args).to.eql([{key1: 'value', key2: 1, key3: true, files: {}}]); + }); + + it('parses body parameter for multipart form data with text file', async () => { + const req = givenRequest({ + url: '/', + headers: { + 'Content-Type': 'multipart/form-data; boundary=--------------------------961406998938770617032718', + }, + payload: '----------------------------961406998938770617032718\r\n' + + 'Content-Disposition: form-data; name="key1"\r\n\r\n' + 'value\r\n' + + '----------------------------961406998938770617032718\r\n' + + 'Content-Disposition: form-data; name="key2"\r\n\r\n' + '1\r\n' + + '----------------------------961406998938770617032718\r\n' + + 'Content-Disposition: form-data; name="key3"\r\n\r\n' + 'true\r\n' + + '----------------------------961406998938770617032718\r\n' + + 'Content-Disposition: form-data; name=""; filename="HelloWorld.txt"\r\n' + + 'Content-Type: text/plain\r\n\r\n' + + 'Hello, LoopBack v4\r\n\r\n' + + '----------------------------961406998938770617032718--\r\n', + }); + + const spec = givenOperationWithRequestBody({ + description: 'data', + content: { + 'multipart/form-data': { + schema: { + type: 'object', + properties: { + key1: {type: 'string'}, + key2: {type: 'number'}, + key3: {type: 'boolean'}, + files: {type: 'object'}, + }, + }, + }, + }, + }); + const route = givenResolvedRoute(spec); + + const args = await parseOperationArgs(req, route); + + const { files } = args[0]; + const fileArray = files['null']; + + // exclude testing file path. + if (fileArray) { + fileArray.forEach(async function(file: File) { + file['path'] = ''; + }); + } + + const testFiles = { + 'null': [ + { + fieldName: null, + originalFilename: 'HelloWorld.txt', + path: '', + headers: { + 'content-disposition': 'form-data; name=""; filename="HelloWorld.txt"', + 'content-type': 'text/plain', + }, + size: 20, + }, + ], + }; + + expect(args).to.eql([{key1: 'value', key2: 1, key3: true, files: testFiles}]); + }); + it('parses path parameters', async () => { const req = givenRequest(); const spec = givenOperationWithParameters([ @@ -58,6 +164,117 @@ describe('operationArgsParser', () => { expect(args).to.eql([{key: 'value'}]); }); + it('parses body parameter for form data', async () => { + const req = givenRequest({ + url: '/', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + payload: 'key=value', + }); + + const spec = givenOperationWithRequestBody({ + description: 'data', + content: { + 'application/x-www-form-urlencoded': {schema: {type: 'object'}}, + }, + }); + const route = givenResolvedRoute(spec); + + const args = await parseOperationArgs(req, route); + + expect(args).to.eql([{key: 'value'}]); + }); + + it('parses body parameter for form data with simple types', async () => { + const req = givenRequest({ + url: '/', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + payload: 'key1=value&key2=1&key3=true', + }); + + const spec = givenOperationWithRequestBody({ + description: 'data', + content: { + 'application/x-www-form-urlencoded': { + schema: { + type: 'object', + properties: { + key1: {type: 'string'}, + key2: {type: 'number'}, + key3: {type: 'boolean'}, + }, + }, + }, + }, + }); + const route = givenResolvedRoute(spec); + + const args = await parseOperationArgs(req, route); + + expect(args).to.eql([{key1: 'value', key2: 1, key3: true}]); + }); + + it('parses body parameter for form data with number[] types', async () => { + const req = givenRequest({ + url: '/', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + payload: 'key=1&key=2', + }); + + const spec = givenOperationWithRequestBody({ + description: 'data', + content: { + 'application/x-www-form-urlencoded': { + schema: { + type: 'object', + properties: { + key: {type: 'array', items: {type: 'number'}}, + }, + }, + }, + }, + }); + const route = givenResolvedRoute(spec); + + const args = await parseOperationArgs(req, route); + + expect(args).to.eql([{key: [1, 2]}]); + }); + + it('parses body parameter for form data with string[] types', async () => { + const req = givenRequest({ + url: '/', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + payload: 'key1=value1&key1=value2', + }); + + const spec = givenOperationWithRequestBody({ + description: 'data', + content: { + 'application/x-www-form-urlencoded': { + schema: { + type: 'object', + properties: { + key1: {type: 'array', items: {type: 'string'}}, + }, + }, + }, + }, + }); + const route = givenResolvedRoute(spec); + + const args = await parseOperationArgs(req, route); + + expect(args).to.eql([{key1: ['value1', 'value2']}]); + }); + context('in:query style:deepObject', () => { it('parses JSON-encoded string value', async () => { const req = givenRequest({