From 40b63077695645087ca5fd01b1663e48f1e4ee3d Mon Sep 17 00:00:00 2001 From: alephseven Date: Tue, 15 Oct 2024 15:50:41 +0530 Subject: [PATCH] Used schemasafe library for validating all OAS version including OAS 3.1.0 for performance --- package.json | 4 +- src/app.ts | 4 +- src/middlewares/schemaValidator.middleware.ts | 227 ++---------------- src/middlewares/validator.ts | 189 +++++++++++++++ src/schemas/configs/app.config.schema.ts | 2 +- src/utils/context.utils.ts | 6 + 6 files changed, 219 insertions(+), 213 deletions(-) create mode 100644 src/middlewares/validator.ts diff --git a/package.json b/package.json index f605082..778bf5f 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,8 @@ "typescript": "^4.7.3" }, "dependencies": { + "@apidevtools/json-schema-ref-parser": "^11.7.2", + "@exodus/schemasafe": "^1.3.0", "amqplib": "^0.10.0", "axios": "^0.26.1", "axios-retry": "^4.0.0", @@ -32,12 +34,12 @@ "cors": "^2.8.5", "dotenv": "^16.4.1", "express": "^4.18.1", - "express-openapi-validator": "^5.1.6", "express-status-monitor": "^1.3.4", "ioredis": "^5.0.6", "libsodium-wrappers": "^0.7.9", "mongodb": "^4.7.0", "node-mocks-http": "^1.15.0", + "openapi-types": "^12.1.3", "request-ip": "^3.3.0", "uuid": "^8.3.2", "winston": "^3.7.2", diff --git a/src/app.ts b/src/app.ts index b423f30..6434091 100644 --- a/src/app.ts +++ b/src/app.ts @@ -12,7 +12,7 @@ import { ClientUtils } from "./utils/client.utils"; import { getConfig } from "./utils/config.utils"; import { GatewayUtils } from "./utils/gateway.utils"; import logger from "./utils/logger.utils"; -import { OpenApiValidatorMiddleware } from "./middlewares/schemaValidator.middleware"; +import { Validator } from "./middlewares/validator"; const app = Express(); @@ -134,7 +134,7 @@ const main = async () => { getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) + getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) ); - await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware(); + await Validator.getInstance().initialize(); logger.info('Initialized openapi validator middleware'); } catch (err) { if (err instanceof Exception) { diff --git a/src/middlewares/schemaValidator.middleware.ts b/src/middlewares/schemaValidator.middleware.ts index 6639449..6b7b9e5 100644 --- a/src/middlewares/schemaValidator.middleware.ts +++ b/src/middlewares/schemaValidator.middleware.ts @@ -1,198 +1,17 @@ -import express, { NextFunction, Request, Response } from "express"; -import * as OpenApiValidator from "express-openapi-validator"; +import { NextFunction, Request, Response } from "express"; import fs from "fs"; import path from "path"; -import YAML from "yaml"; -import * as httpMocks from "node-mocks-http"; -import { v4 as uuid_v4 } from "uuid"; import { Exception, ExceptionType } from "../models/exception.model"; import { Locals } from "../interfaces/locals.interface"; import { getConfig } from "../utils/config.utils"; -import { OpenAPIV3 } from "express-openapi-validator/dist/framework/types"; import logger from "../utils/logger.utils"; import { AppMode } from "../schemas/configs/app.config.schema"; import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; -import { - RequestActions, - ResponseActions -} from "../schemas/configs/actions.app.config.schema"; +import { Validator } from "./validator"; import { validationFailHandler } from "../utils/validations.utils"; const specFolder = 'schemas'; -export class OpenApiValidatorMiddleware { - private static instance: OpenApiValidatorMiddleware; - private static cachedOpenApiValidator: { - [filename: string]: { - count: number, - requestHandler: express.RequestHandler[], - apiSpec: OpenAPIV3.Document - } - } = {}; - private static cachedFileLimit: number; - - private constructor() { - OpenApiValidatorMiddleware.cachedFileLimit = getConfig().app.openAPIValidator?.cachedFileLimit || 20; - } - - public static getInstance(): OpenApiValidatorMiddleware { - if (!OpenApiValidatorMiddleware.instance) { - OpenApiValidatorMiddleware.instance = new OpenApiValidatorMiddleware(); - } - return OpenApiValidatorMiddleware.instance; - } - - private getApiSpec(specFile: string): OpenAPIV3.Document { - const apiSpecYAML = fs.readFileSync(specFile, "utf8"); - const apiSpec = YAML.parse(apiSpecYAML); - return apiSpec; - }; - - public async initOpenApiMiddleware(): Promise { - try { - let fileToCache = getConfig().app?.openAPIValidator?.initialFilesToCache; - let fileNames, noOfFileToCache = 0; - const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit; - logger.info(`OpenAPIValidator Total Cache capacity ${cachedFileLimit}`); - if (fileToCache) { - fileNames = fileToCache.split(/\s*,\s*/).map(item => item.trim()); - logger.info(`OpenAPIValidator Init no of files to cache: ${fileNames?.length}`); - noOfFileToCache = fileNames.length; - } else { - const files = fs.readdirSync(specFolder); - fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml'))); - noOfFileToCache = Math.min(fileNames.length, 3); //If files to cache is not found in env then we will cache just three file - } - noOfFileToCache = Math.min(noOfFileToCache, cachedFileLimit); - console.log('Cache total files: ', noOfFileToCache); - - for (let i = 0; i < noOfFileToCache; i++) { - const file = `${specFolder}/${fileNames[i]}`; - if (!OpenApiValidatorMiddleware.cachedOpenApiValidator[file]) { - logger.info(`Intially cache Not found loadApiSpec file. Loading.... ${file}`); - const apiSpec = this.getApiSpec(file); - const requestHandler = OpenApiValidator.middleware({ - apiSpec, - validateRequests: true, - validateResponses: false, - $refParser: { - mode: "dereference" - } - }) - OpenApiValidatorMiddleware.cachedOpenApiValidator[file] = { - apiSpec, - count: 0, - requestHandler - } - await initializeOpenApiValidatorCache(requestHandler); - } - } - } catch (err) { - logger.error('Error in initializing open API middleware', err); - } - } - - public getOpenApiMiddleware(specFile: string): express.RequestHandler[] { - try { - let requestHandler: express.RequestHandler[]; - if (OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]) { - const cachedValidator = OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]; - cachedValidator.count = cachedValidator.count > 1000 ? cachedValidator.count : cachedValidator.count + 1; - logger.info(`Cache found for spec ${specFile}`); - requestHandler = cachedValidator.requestHandler; - } else { - const cashedSpec = Object.entries(OpenApiValidatorMiddleware.cachedOpenApiValidator); - const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit; - if (cashedSpec.length >= cachedFileLimit) { - const specWithLeastCount = cashedSpec.reduce((minEntry, currentEntry) => { - return currentEntry[1].count < minEntry[1].count ? currentEntry : minEntry; - }) || cashedSpec[0]; - logger.info(`Cache count reached limit. Deleting from cache.... ${specWithLeastCount[0]}`); - delete OpenApiValidatorMiddleware.cachedOpenApiValidator[specWithLeastCount[0]]; - } - logger.info(`Cache Not found loadApiSpec file. Loading.... ${specFile}`); - const apiSpec = this.getApiSpec(specFile); - OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile] = { - apiSpec, - count: 1, - requestHandler: OpenApiValidator.middleware({ - apiSpec, - validateRequests: true, - validateResponses: false, - $refParser: { - mode: "dereference" - } - }) - } - requestHandler = OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile].requestHandler; - } - const cacheStats = Object.entries(OpenApiValidatorMiddleware.cachedOpenApiValidator).map((cache) => { - return { - count: cache[1].count, - specFile: cache[0] - } - }); - console.table(cacheStats); - return requestHandler; - } catch (err) { - logger.error('Error in getOpenApiMiddleware', err); - return [] - } - }; -} - -const initializeOpenApiValidatorCache = async (stack: any) => { - try { - let actions: string[] = []; - if ( - (getConfig().app.mode === AppMode.bap && - getConfig().app.gateway.mode === GatewayMode.client) || - (getConfig().app.mode === AppMode.bpp && - getConfig().app.gateway.mode === GatewayMode.network) - ) { - actions = Object.keys(RequestActions); - } else { - actions = Object.keys(ResponseActions); - } - - actions.forEach((action) => { - const mockRequest = (body: any) => { - const req = httpMocks.createRequest({ - method: "POST", - url: `/${action}`, - headers: { - "Content-Type": "application/json", - Authorization: uuid_v4() - }, - body: body - }); - - req.app = { - enabled: (setting: any) => { - if ( - setting === "strict routing" || - setting === "case sensitive routing" - ) { - return true; - } - return false; - } - } as any; - return req; - }; - - const reqObj = mockRequest({ - context: { action: `${action}` }, - message: {} - }); - - walkSubstack(stack, reqObj, {}, () => { - return; - }, false); - }); - } catch (error: any) { } -}; - export const schemaErrorHandler = ( err: any, req: Request, @@ -234,28 +53,6 @@ export const schemaErrorHandler = ( } }; -const walkSubstack = function ( - stack: any, - req: any, - res: any, - next: NextFunction, - reportError = true -) { - if (typeof stack === "function") { - stack = [stack]; - } - const walkStack = function (i: any, err?: any) { - if (err && reportError) { - return schemaErrorHandler(err, req, res, next); - } - if (i >= stack.length) { - return next(); - } - stack[i](req, res, walkStack.bind(null, i + 1)); - }; - walkStack(0); -}; - export const openApiValidatorMiddleware = async ( req: Request, res: Response<{}, Locals>, @@ -265,7 +62,7 @@ export const openApiValidatorMiddleware = async ( ? req?.body?.context?.core_version : req?.body?.context?.version; let specFile = `${specFolder}/core_${version}.yaml`; - + let specFileName = `core_${version}.yaml`; if (getConfig().app.useLayer2Config) { let doesLayer2ConfigExist = false; let layer2ConfigFilename = `${req?.body?.context?.domain}_${version}.yaml`; @@ -280,7 +77,10 @@ export const openApiValidatorMiddleware = async ( } catch (error) { doesLayer2ConfigExist = false; } - if (doesLayer2ConfigExist) specFile = `${specFolder}/${layer2ConfigFilename}`; + if (doesLayer2ConfigExist) { + specFile = `${specFolder}/${layer2ConfigFilename}`; + specFileName = layer2ConfigFilename; + } else { if (getConfig().app.mandateLayer2Config) { const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration` @@ -295,6 +95,15 @@ export const openApiValidatorMiddleware = async ( } } } - const openApiValidator = OpenApiValidatorMiddleware.getInstance().getOpenApiMiddleware(specFile); - walkSubstack([...openApiValidator], req, res, next); + const validatorInstance = Validator.getInstance(); + const openApiValidator = await validatorInstance.getValidationMiddleware(specFile, specFileName); + // Call the openApiValidator and handle the response + await openApiValidator(req, res, (err: any) => { + if (err) { + schemaErrorHandler(err, req, res, next); + } else { + logger.info('Validation Success'); + next(); + } + }); }; diff --git a/src/middlewares/validator.ts b/src/middlewares/validator.ts new file mode 100644 index 0000000..b6c6491 --- /dev/null +++ b/src/middlewares/validator.ts @@ -0,0 +1,189 @@ +import fs from 'fs'; +import YAML from 'yaml'; +import { OpenAPIV3 } from 'openapi-types'; +import $RefParser from '@apidevtools/json-schema-ref-parser'; +import path from "path"; +import logger from '../utils/logger.utils'; +import { getConfig } from '../utils/config.utils'; +import { validator, ValidatorOptions } from '@exodus/schemasafe'; +import { Exception, ExceptionType } from '../models/exception.model'; +const specFolder = 'schemas'; + +export class Validator { + private static instance: Validator; + private static schemaCache: { + [keyName: string]: { + count: number, + requestHandler: Function + } + } = {}; + private initialized: boolean = false; + private constructor() {} + + public static getInstance(): Validator { + if (!Validator.instance) { + Validator.instance = new Validator(); + } + return Validator.instance; + } + + async initialize() { + if (this.initialized) return; + console.time('SchemaValidation'); + await this.compileEachSpecFiles(); + console.timeEnd('SchemaValidation'); + this.initialized = true; + } + + private getApiSpec(specFile: string): OpenAPIV3.Document { + const apiSpecYAML = fs.readFileSync(specFile, "utf8"); + const apiSpec = YAML.parse(apiSpecYAML); + return apiSpec; + }; + + async compileEachSpecFiles() { + const cachedFileLimit: number = getConfig().app?.openAPIValidator?.cachedFileLimit || 3; + logger.info(`OpenAPIValidator Cache count ${cachedFileLimit}`); + const files = fs.readdirSync(specFolder); + const fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml'))); + logger.info(`OpenAPIValidator loaded spec files ${fileNames}`); + let i = 0; + logger.info(`Total file: ${fileNames.length}`); + const noOfSpecToCompile = Math.min(cachedFileLimit, fileNames.length); + logger.info(`Compiling ${noOfSpecToCompile} spec file`); + while (i < noOfSpecToCompile) { + const file = `${specFolder}/${fileNames[i]}`; + const options = { + continueOnError: true, // Continue dereferencing despite errors + }; + let dereferencedSpec: any; + const spec = this.getApiSpec(file); + + try { + dereferencedSpec = await $RefParser.dereference(spec, options) as OpenAPIV3.Document; + } catch (error) { + console.error('Dereferencing error:', error); + } + + try { + await this.compileSchemas(dereferencedSpec, fileNames[i]); + } catch (error) { + logger.error(`Error compiling doc: ${error}`); + } + i++; + } + logger.info(`Schema cache size: ${Object.keys(Validator.schemaCache).length}`); + const cacheStats = Object.entries(Validator.schemaCache).map((cache) => { + return { + count: cache[1].count, + specFile: cache[0] + } + }); + console.table(cacheStats); + + } + + async compileSchemas(spec: OpenAPIV3.Document, file: string, schemaPath?: string | null | undefined, schemaMethod?: string | null | undefined) { + const regex = /\.(yml|yaml)$/; + const fileName = file.split(regex)[0]; + logger.info(`OpenAPIValidator compile schema fileName: ${fileName}`); + + for (const path of Object.keys(spec.paths)) { + const methods: any = spec.paths[path]; + if (!schemaPath || schemaPath === path) { + for (const method of Object.keys(methods)) { + if (!schemaMethod || schemaMethod === method) { + const operation = methods[method]; + const key = `${fileName}-${path}-${method}`; + + const options: ValidatorOptions = { + mode: 'lax', + includeErrors: true, // Include errors in the output + allErrors: true, // Report all validation errors + contentValidation: true, // Validate content based on formats, + $schemaDefault: 'http://json-schema.org/draft/2020-12/schema', // Specify the schema version + }; + if (!Validator.schemaCache[key]) { + try { + const parse = validator(operation.requestBody?.content['application/json']?.schema, options) + Validator.schemaCache[key] = { + count: 0, + requestHandler: parse + } + logger.info(`Schema compiled and cached for ${key}`); + } catch (error: any) { + logger.error(`Error compiling schema for ${key}: ${error.message}`); + } + } + } + } + } + } + + } + + async getValidationMiddleware(specFile: string, specFileName: string) { + return async (req: any, res: any, next: any) => { + logger.info(`Spec file: ${specFile}`); + const regex = /\.(yml|yaml)$/; + const fileName = specFileName.split(regex)[0]; + logger.info(`File name: ${specFile}`); + const action = `/${req.body.context.action}`; + const method = req.method.toLowerCase(); + const requestKey = `${fileName}-${action}-${method}`; + const validateKey = Validator.schemaCache[requestKey]; + if (validateKey) { + logger.info(`Schemasafe Validation Cache HIT for ${specFileName}`); + const validate: any = validateKey.requestHandler; + try { + const validationResult = validate(req.body); + if (!validationResult) { + throw new Exception( + ExceptionType.OpenApiSchema_ParsingError, + 'Schema validation failed', + 400, + validate.errors + ); + } + validateKey.count = validateKey.count ? validateKey.count + 1 : 1; + console.table([{key: requestKey, count: Validator.schemaCache[requestKey].count}]); + } catch (error) { + return next(error); + } + } else { + const cashedSpec = Object.entries(Validator.schemaCache); + const cachedFileLimit: number = getConfig().app?.openAPIValidator?.cacheSizeLimit || 100; + if (cashedSpec.length >= cachedFileLimit) { + const specWithLeastCount = cashedSpec.reduce((minEntry, currentEntry) => { + return currentEntry[1].count < minEntry[1].count ? currentEntry : minEntry; + }) || cashedSpec[0]; + logger.info(`Cache count reached limit. Deleting from cache.... ${specWithLeastCount[0]}`); + delete Validator.schemaCache[specWithLeastCount[0]]; + } + logger.info(`Schemasafe Validation Cache miss for ${specFileName}`); + const apiSpecYAML = this.getApiSpec(specFile); + const dereferencedSpec = await $RefParser.dereference(apiSpecYAML) as OpenAPIV3.Document; + + try { + await this.compileSchemas(dereferencedSpec, specFileName, action, method); + const validateKey = Validator.schemaCache[requestKey]; + const validate: any = validateKey.requestHandler; + const validationResult = validate(req.body); + if (!validationResult) { + throw new Exception( + ExceptionType.OpenApiSchema_ParsingError, + 'Schema validation failed', + 400, + validate.errors + ); + } + validateKey.count = validateKey.count ? validateKey.count + 1 : 1; + } catch (error) { + logger.error(`Error compiling doc: ${error}`); + return next(error); + } + } + next(); + }; + } +} \ No newline at end of file diff --git a/src/schemas/configs/app.config.schema.ts b/src/schemas/configs/app.config.schema.ts index 25f656d..96f3ae0 100644 --- a/src/schemas/configs/app.config.schema.ts +++ b/src/schemas/configs/app.config.schema.ts @@ -59,7 +59,7 @@ export const appConfigSchema = z.object({ sharedKeyForWebhookHMAC: z.string().optional(), openAPIValidator: z.object({ cachedFileLimit: z.number().optional(), - initialFilesToCache: z.string().optional() + cacheSizeLimit: z.number().optional() }).optional() }); diff --git a/src/utils/context.utils.ts b/src/utils/context.utils.ts index 6ff39cd..254cb36 100644 --- a/src/utils/context.utils.ts +++ b/src/utils/context.utils.ts @@ -59,6 +59,12 @@ export const bapContextBuilder = async ( bapContext )}\n` ); + //remove all keys having null or undefined values for validation + for (const key in bapContext) { + if (bapContext[key] === undefined || bapContext[key] === null) { + delete bapContext[key]; + } + } return bapContext; };