diff --git a/src/app.ts b/src/app.ts index 600918e..7db4d09 100644 --- a/src/app.ts +++ b/src/app.ts @@ -136,7 +136,7 @@ const main = async () => { getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) ); await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware(); - await Validator.getInstance(false).initialize() + await Validator.getInstance().initialize(); logger.info('Initialized openapi validator middleware'); } catch (err) { if (err instanceof Exception) { diff --git a/src/middlewares/schemaValidator.middleware.ts b/src/middlewares/schemaValidator.middleware.ts index 0c461ba..f2b59d9 100644 --- a/src/middlewares/schemaValidator.middleware.ts +++ b/src/middlewares/schemaValidator.middleware.ts @@ -283,9 +283,9 @@ export const openApiValidatorMiddleware = async ( const apiSpecYAML = fs.readFileSync(specFile, "utf8"); const apiSpec = YAML.parse(apiSpecYAML); if (apiSpec.openapi === '3.1.0') { - const ajvValidatorInstance = Validator.getInstance(false); + const ajvValidatorInstance = Validator.getInstance(); const openApiValidator = ajvValidatorInstance.getValidationMiddleware(); - openApiValidator(req, res, () => { + (await openApiValidator)(req, res, () => { console.log('Validation Success'); next() }); diff --git a/src/middlewares/schemaValidatorAjv.middleware.ts b/src/middlewares/schemaValidatorAjv.middleware.ts index 932af89..c2c7d92 100644 --- a/src/middlewares/schemaValidatorAjv.middleware.ts +++ b/src/middlewares/schemaValidatorAjv.middleware.ts @@ -7,110 +7,37 @@ import $RefParser from '@apidevtools/json-schema-ref-parser'; import path from "path"; import logger from '../utils/logger.utils'; import { NextFunction, Request, Response } from 'express'; -import { Worker } from 'worker_threads'; import { Locals } from "../interfaces/locals.interface"; +import { getConfig } from '../utils/config.utils'; +import { Exception, ExceptionType } from '../models/exception.model'; const specFolder = 'schemas'; export class Validator { private static instance: Validator; private ajv: Ajv; private schemaCache: Map; private initialized: boolean = false; - private shouldRunWorker: boolean = false; private constructor() { this.ajv = new Ajv({ allErrors: true, coerceTypes: true, useDefaults: true, strict: false }); addFormats(this.ajv); this.schemaCache = new Map(); } - public static getInstance(shouldRunWorker: boolean): Validator { + public static getInstance(): Validator { + if (!Validator.instance) { Validator.instance = new Validator(); } - Validator.instance.shouldRunWorker = shouldRunWorker; return Validator.instance; } async initialize() { if (this.initialized) return; console.time('SchemaValidation'); - if (this.shouldRunWorker) { - await this.initializeWorker(); - } else { - console.log('Running in main thread...'); - await this.compileEachSpecFiles(); - } - + console.log('Running in main thread...'); + await this.compileEachSpecFiles(); console.timeEnd('SchemaValidation'); this.initialized = true; } - async initializeWorker() { - console.log('Running in worker thread...'); - const files = fs.readdirSync(specFolder); - const fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml'))); - console.log('File names: ', fileNames); - // for (const specPath of fileNames) { - // const serializedEntries: any = await this.runWorker(specPath); - // //console.log('Serialized entries: ', serializedEntries); - - // let deserializedEntries: any; - // try { - // deserializedEntries = JSON.parse(serializedEntries); - // } catch (error) { - // console.error('Error deserializing entries:', error); - // continue; // Skip this entry if deserialization fails - // } - - // deserializedEntries.forEach(([key, value]: [string, any]) => { - // this.schemaCache.set(key, value); - // }); - // } - const workerPromises = fileNames.map(specPath => this.runWorker(specPath)); - let schemaEntries: any = await Promise.all(workerPromises); - console.log('Schema entries: ', typeof schemaEntries); - if (typeof schemaEntries == 'string') { - schemaEntries = JSON.parse(schemaEntries); - } - schemaEntries.forEach((entries: any) => { - let deserializedCache; - console.log('Enteries type : ', typeof entries); - if (typeof entries == 'string') { - console.log('XX'); - logger.info(`Parsing: , ${entries}`); - deserializedCache = JSON.parse(entries); - } - console.log('Decentralized cache type: ', typeof deserializedCache); - - deserializedCache.forEach(([key, value]: [string, any]) => { - this.schemaCache.set(key, value); - }); - }); - } - - private runWorker(specPath: string): Promise<[string, any][]> { - return new Promise((resolve, reject) => { - const worker = new Worker(path.resolve(__dirname, 'schema-compiler-worker.js'), { - workerData: { - specPath, - path: './schema-compiler-worker.ts' - } - }); - - worker.on('message', (message) => { - try { - // Assuming message is already serialized, directly resolve it - resolve(message); - } catch (error) { - reject(error); - } - }); - - worker.on('exit', (code) => { - if (code !== 0) { - reject(new Error(`Worker stopped with exit code ${code}`)); - } - }); - }); - } private getApiSpec(specFile: string): OpenAPIV3.Document { const apiSpecYAML = fs.readFileSync(specFile, "utf8"); @@ -119,7 +46,7 @@ export class Validator { }; async compileEachSpecFiles() { - const cachedFileLimit: number = 20; + const cachedFileLimit: number = 4; const files = fs.readdirSync(specFolder); const fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml'))); logger.info(`OpenAPIValidator loaded spec files ${fileNames}`); @@ -132,7 +59,6 @@ export class Validator { }; let dereferencedSpec: any; dereferencedSpec = await $RefParser.dereference(this.getApiSpec(file), options) as OpenAPIV3.Document; - //console.log('Dereferenced spec file: ', JSON.stringify(dereferencedSpec)); try { @@ -140,21 +66,18 @@ export class Validator { } catch (error) { console.log('Error derefencing doc: ', error); } - //const dereferencedSpec = await $RefParser.dereference(this.getApiSpec(file), options) as OpenAPIV3.Document; } console.log('Schema cache size: ', this.schemaCache.size); for (const [key, _] of this.schemaCache) { - //logger.info(`Set all cache for validation key and its value : ${key}`); + logger.info(`Set all cache for validation key and its value : ${key}`); console.log(`Set all cache for validation key and its value : ${key}`); } } private async compileSchemas(spec: OpenAPIV3.Document, file: string) { - //logger.info(`OpenAPIValidator compile schema file: ${file}`); - //logger.info(`OpenAPIValidator compile schema specfile: ${spec}`); const regex = /\.(yml|yaml)$/; const fileName = file.split(regex)[0]; logger.info(`OpenAPIValidator compile schema fileName: ${fileName}`); @@ -164,17 +87,17 @@ export class Validator { const operation = methods[method]; // Compile request body schema + const bodyKey = `${fileName}-${path}-${method}-requestBody`; const requestBodySchema = operation.requestBody && (operation.requestBody as any).content['application/json'].schema; - if (requestBodySchema) { - const key = `${fileName}-${path}-${method}-requestBody`; - this.schemaCache.set(key, this.ajv.compile(requestBodySchema)); + if (!this.schemaCache.has(bodyKey) && requestBodySchema) { + this.schemaCache.set(bodyKey, this.ajv.compile(requestBodySchema)); } // Compile query parameters schema + const queryKey = `${fileName}-${path}-${method}-queryParameters`; const queryParameters = (operation.parameters || []).filter((param: any) => param.in === 'query'); - if (queryParameters.length > 0) { - const key = `${fileName}-${path}-${method}-queryParameters`; - this.schemaCache.set(key, this.ajv.compile({ + if (!this.schemaCache.has(queryKey) && queryParameters.length) { + this.schemaCache.set(queryKey, this.ajv.compile({ type: 'object', properties: queryParameters.reduce((acc: { [x: string]: any; }, param: { name: string | number; schema: any; }) => { acc[param.name] = param.schema; return acc; @@ -184,9 +107,9 @@ export class Validator { // Compile headers schema const headers = (operation.parameters || []).filter((param: any) => param.in === 'header'); - if (headers.length > 0) { - const key = `${fileName}-${path}-${method}-headers`; - this.schemaCache.set(key, this.ajv.compile({ + const headerKey = `${fileName}-${path}-${method}-headers`; + if (!this.schemaCache.has(headerKey) && headers.length) { + this.schemaCache.set(headerKey, this.ajv.compile({ type: 'object', properties: headers.reduce((acc: { [x: string]: any; }, param: { name: string | number; schema: any; }) => { acc[param.name] = param.schema; return acc; @@ -204,8 +127,8 @@ export class Validator { }); } - getValidationMiddleware() { - return (req: Request, + async getValidationMiddleware() { + return async (req: Request, res: Response<{}, Locals>, next: NextFunction) => { let version = req?.body?.context?.core_version @@ -215,10 +138,46 @@ export class Validator { domain = domain.replace(/:/g, '_'); const formattedVersion = `${domain.trim()}_${version.trim()}`; console.log('Formatted version: ', formattedVersion); - const action = `/${req?.body?.context?.action}`; const method = req.method.toLowerCase(); - + let specFile = `${specFolder}/core_${version}.yaml`; + let specFileName = `core_${version}.yaml`; + for (const [key, _] of this.schemaCache) { + //logger.info(`Set all cache for validation key and its value : ${key}`); + console.log(`Cache key: ${key}`); + } + if (getConfig().app.useLayer2Config) { + let doesLayer2ConfigExist = false; + let layer2ConfigFilename = `${req?.body?.context?.domain}_${version}.yaml`; + let specialCharsRe = /[:\/]/gi; + layer2ConfigFilename = layer2ConfigFilename.replace(specialCharsRe, "_"); + try { + doesLayer2ConfigExist = ( + await fs.promises.readdir( + `${path.join(path.resolve(__dirname, "../../"))}/${specFolder}` + ) + ).includes(layer2ConfigFilename); + } catch (error) { + doesLayer2ConfigExist = false; + } + if (doesLayer2ConfigExist) { + specFile = `${specFolder}/${layer2ConfigFilename}`; + specFileName = layer2ConfigFilename; + } + else { + if (getConfig().app.mandateLayer2Config) { + const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration` + logger.error(message); + return next( + new Exception( + ExceptionType.Config_AppConfig_Layer2_Missing, + message, + 422 + ) + ); + } + } + } // Validate request body const requestBodyKey = `${formattedVersion}-${action}-${method}-requestBody`; logger.info(`requestBodyKey for incoming req: ${requestBodyKey}`) @@ -228,18 +187,27 @@ export class Validator { return res.status(400).json({ error: validateRequestBody.errors }); } } else { - //compile schema - //Find the spec file - //load the spec file + console.log(`AGV Validation Cache miss for ${specFileName} and request body: ${requestBodyKey}`); + const apiSpecYAML = this.getApiSpec(specFile); //parse and destructure the spec file - //call this.compileSchema(specFile) + const options = { + continueOnError: true, // Continue dereferencing despite errors + }; + let dereferencedSpec: any; + dereferencedSpec = await $RefParser.dereference(apiSpecYAML, options) as OpenAPIV3.Document; + + try { + await this.compileSchemas(dereferencedSpec, specFileName); + } catch (error) { + console.log('Error derefencing doc: ', error); + } const validateRequestBody: any = this.schemaCache.get(requestBodyKey); if (!validateRequestBody(req.body)) { return res.status(400).json({ error: validateRequestBody.errors }); } } - // Validate query parameters + //Validate query parameters const queryParametersKey = `${formattedVersion}-${action}-${method}-queryParameters`; if (this.schemaCache.has(queryParametersKey)) { const validateQueryParameters: any = this.schemaCache.get(queryParametersKey); @@ -249,9 +217,21 @@ export class Validator { } else { //compile schema //Find the spec file - //load the spec file + console.log(`AGV Validation Cache miss for ${specFileName} and query-param-key: ${queryParametersKey}`); + const apiSpecYAML = this.getApiSpec(specFile); //parse and destructure the spec file - //call this.compileSchema(specFile) + const options = { + continueOnError: true, // Continue dereferencing despite errors + }; + let dereferencedSpec: any; + dereferencedSpec = await $RefParser.dereference(apiSpecYAML, options) as OpenAPIV3.Document; + //console.log('Dereferenced spec file: ', JSON.stringify(dereferencedSpec)); + + try { + await this.compileSchemas(dereferencedSpec, specFileName); + } catch (error) { + console.log('Error derefencing doc: ', error); + } const validateRequestBody: any = this.schemaCache.get(requestBodyKey); if (!validateRequestBody(req.body)) { return res.status(400).json({ error: validateRequestBody.errors }); @@ -268,18 +248,28 @@ export class Validator { } else { //compile schema //Find the spec file - //load the spec file + console.log(`AGV Validation Cache miss for ${specFileName} and header-key: ${headersKey}`); + const apiSpecYAML = this.getApiSpec(specFile); //parse and destructure the spec file - //call this.compileSchema(specFile) + const options = { + continueOnError: true, // Continue dereferencing despite errors + }; + let dereferencedSpec: any; + dereferencedSpec = await $RefParser.dereference(apiSpecYAML, options) as OpenAPIV3.Document; + //console.log('Dereferenced spec file: ', JSON.stringify(dereferencedSpec)); + + try { + await this.compileSchemas(dereferencedSpec, specFileName); + } catch (error) { + console.log('Error derefencing doc: ', error); + } const validateRequestBody: any = this.schemaCache.get(requestBodyKey); if (!validateRequestBody(req.body)) { return res.status(400).json({ error: validateRequestBody.errors }); } } - next(); }; } } -