Skip to content

Commit

Permalink
Removed worker support in AGV and added support for openapi 3.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@rahul-eminds
rahul-eminds committed Oct 1, 2024
1 parent cab6fd6 commit 97c51fa
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 112 deletions.
2 changes: 1 addition & 1 deletion src/app.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ const main = async () => {
getConfig().app.gateway.mode.toLocaleUpperCase().substring(1)
);
await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware();
await Validator.getInstance(false).initialize()
await Validator.getInstance().initialize();
logger.info('Initialized openapi validator middleware');
} catch (err) {
if (err instanceof Exception) {
Expand Down
4 changes: 2 additions & 2 deletions src/middlewares/schemaValidator.middleware.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,9 +283,9 @@ export const openApiValidatorMiddleware = async (
const apiSpecYAML = fs.readFileSync(specFile, "utf8");
const apiSpec = YAML.parse(apiSpecYAML);
if (apiSpec.openapi === '3.1.0') {
const ajvValidatorInstance = Validator.getInstance(false);
const ajvValidatorInstance = Validator.getInstance();
const openApiValidator = ajvValidatorInstance.getValidationMiddleware();
openApiValidator(req, res, () => {
(await openApiValidator)(req, res, () => {
console.log('Validation Success');
next()
});
Expand Down
208 changes: 99 additions & 109 deletions src/middlewares/schemaValidatorAjv.middleware.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,110 +7,37 @@ import $RefParser from '@apidevtools/json-schema-ref-parser';
import path from "path";
import logger from '../utils/logger.utils';
import { NextFunction, Request, Response } from 'express';
import { Worker } from 'worker_threads';
import { Locals } from "../interfaces/locals.interface";
import { getConfig } from '../utils/config.utils';
import { Exception, ExceptionType } from '../models/exception.model';
const specFolder = 'schemas';
export class Validator {
private static instance: Validator;
private ajv: Ajv;
private schemaCache: Map<string, ValidateFunction>;
private initialized: boolean = false;
private shouldRunWorker: boolean = false;
private constructor() {
this.ajv = new Ajv({ allErrors: true, coerceTypes: true, useDefaults: true, strict: false });
addFormats(this.ajv);
this.schemaCache = new Map<string, ValidateFunction>();
}

public static getInstance(shouldRunWorker: boolean): Validator {
public static getInstance(): Validator {

if (!Validator.instance) {
Validator.instance = new Validator();
}
Validator.instance.shouldRunWorker = shouldRunWorker;
return Validator.instance;
}

async initialize() {
if (this.initialized) return;
console.time('SchemaValidation');
if (this.shouldRunWorker) {
await this.initializeWorker();
} else {
console.log('Running in main thread...');
await this.compileEachSpecFiles();
}

console.log('Running in main thread...');
await this.compileEachSpecFiles();
console.timeEnd('SchemaValidation');
this.initialized = true;
}
async initializeWorker() {
console.log('Running in worker thread...');
const files = fs.readdirSync(specFolder);
const fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml')));
console.log('File names: ', fileNames);
// for (const specPath of fileNames) {
// const serializedEntries: any = await this.runWorker(specPath);
// //console.log('Serialized entries: ', serializedEntries);

// let deserializedEntries: any;
// try {
// deserializedEntries = JSON.parse(serializedEntries);
// } catch (error) {
// console.error('Error deserializing entries:', error);
// continue; // Skip this entry if deserialization fails
// }

// deserializedEntries.forEach(([key, value]: [string, any]) => {
// this.schemaCache.set(key, value);
// });
// }
const workerPromises = fileNames.map(specPath => this.runWorker(specPath));
let schemaEntries: any = await Promise.all(workerPromises);
console.log('Schema entries: ', typeof schemaEntries);
if (typeof schemaEntries == 'string') {
schemaEntries = JSON.parse(schemaEntries);
}
schemaEntries.forEach((entries: any) => {
let deserializedCache;
console.log('Enteries type : ', typeof entries);
if (typeof entries == 'string') {
console.log('XX');
logger.info(`Parsing: , ${entries}`);
deserializedCache = JSON.parse(entries);
}
console.log('Decentralized cache type: ', typeof deserializedCache);

deserializedCache.forEach(([key, value]: [string, any]) => {
this.schemaCache.set(key, value);
});
});
}

private runWorker(specPath: string): Promise<[string, any][]> {
return new Promise((resolve, reject) => {
const worker = new Worker(path.resolve(__dirname, 'schema-compiler-worker.js'), {
workerData: {
specPath,
path: './schema-compiler-worker.ts'
}
});

worker.on('message', (message) => {
try {
// Assuming message is already serialized, directly resolve it
resolve(message);
} catch (error) {
reject(error);
}
});

worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`));
}
});
});
}

private getApiSpec(specFile: string): OpenAPIV3.Document {
const apiSpecYAML = fs.readFileSync(specFile, "utf8");
Expand All @@ -119,7 +46,7 @@ export class Validator {
};

async compileEachSpecFiles() {
const cachedFileLimit: number = 20;
const cachedFileLimit: number = 4;
const files = fs.readdirSync(specFolder);
const fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml')));
logger.info(`OpenAPIValidator loaded spec files ${fileNames}`);
Expand All @@ -132,29 +59,25 @@ export class Validator {
};
let dereferencedSpec: any;
dereferencedSpec = await $RefParser.dereference(this.getApiSpec(file), options) as OpenAPIV3.Document;
//console.log('Dereferenced spec file: ', JSON.stringify(dereferencedSpec));

try {

await this.compileSchemas(dereferencedSpec, fileNames[i]);
} catch (error) {
console.log('Error derefencing doc: ', error);
}
//const dereferencedSpec = await $RefParser.dereference(this.getApiSpec(file), options) as OpenAPIV3.Document;


}
console.log('Schema cache size: ', this.schemaCache.size);
for (const [key, _] of this.schemaCache) {
//logger.info(`Set all cache for validation key and its value : ${key}`);
logger.info(`Set all cache for validation key and its value : ${key}`);
console.log(`Set all cache for validation key and its value : ${key}`);
}

}

private async compileSchemas(spec: OpenAPIV3.Document, file: string) {
//logger.info(`OpenAPIValidator compile schema file: ${file}`);
//logger.info(`OpenAPIValidator compile schema specfile: ${spec}`);
const regex = /\.(yml|yaml)$/;
const fileName = file.split(regex)[0];
logger.info(`OpenAPIValidator compile schema fileName: ${fileName}`);
Expand All @@ -164,17 +87,17 @@ export class Validator {
const operation = methods[method];

// Compile request body schema
const bodyKey = `${fileName}-${path}-${method}-requestBody`;
const requestBodySchema = operation.requestBody && (operation.requestBody as any).content['application/json'].schema;
if (requestBodySchema) {
const key = `${fileName}-${path}-${method}-requestBody`;
this.schemaCache.set(key, this.ajv.compile(requestBodySchema));
if (!this.schemaCache.has(bodyKey) && requestBodySchema) {
this.schemaCache.set(bodyKey, this.ajv.compile(requestBodySchema));
}

// Compile query parameters schema
const queryKey = `${fileName}-${path}-${method}-queryParameters`;
const queryParameters = (operation.parameters || []).filter((param: any) => param.in === 'query');
if (queryParameters.length > 0) {
const key = `${fileName}-${path}-${method}-queryParameters`;
this.schemaCache.set(key, this.ajv.compile({
if (!this.schemaCache.has(queryKey) && queryParameters.length) {
this.schemaCache.set(queryKey, this.ajv.compile({
type: 'object', properties: queryParameters.reduce((acc: { [x: string]: any; }, param: { name: string | number; schema: any; }) => {
acc[param.name] = param.schema;
return acc;
Expand All @@ -184,9 +107,9 @@ export class Validator {

// Compile headers schema
const headers = (operation.parameters || []).filter((param: any) => param.in === 'header');
if (headers.length > 0) {
const key = `${fileName}-${path}-${method}-headers`;
this.schemaCache.set(key, this.ajv.compile({
const headerKey = `${fileName}-${path}-${method}-headers`;
if (!this.schemaCache.has(headerKey) && headers.length) {
this.schemaCache.set(headerKey, this.ajv.compile({
type: 'object', properties: headers.reduce((acc: { [x: string]: any; }, param: { name: string | number; schema: any; }) => {
acc[param.name] = param.schema;
return acc;
Expand All @@ -204,8 +127,8 @@ export class Validator {
});
}

getValidationMiddleware() {
return (req: Request,
async getValidationMiddleware() {
return async (req: Request,
res: Response<{}, Locals>,
next: NextFunction) => {
let version = req?.body?.context?.core_version
Expand All @@ -215,10 +138,46 @@ export class Validator {
domain = domain.replace(/:/g, '_');
const formattedVersion = `${domain.trim()}_${version.trim()}`;
console.log('Formatted version: ', formattedVersion);

const action = `/${req?.body?.context?.action}`;
const method = req.method.toLowerCase();

let specFile = `${specFolder}/core_${version}.yaml`;
let specFileName = `core_${version}.yaml`;
for (const [key, _] of this.schemaCache) {
//logger.info(`Set all cache for validation key and its value : ${key}`);
console.log(`Cache key: ${key}`);
}
if (getConfig().app.useLayer2Config) {
let doesLayer2ConfigExist = false;
let layer2ConfigFilename = `${req?.body?.context?.domain}_${version}.yaml`;
let specialCharsRe = /[:\/]/gi;
layer2ConfigFilename = layer2ConfigFilename.replace(specialCharsRe, "_");
try {
doesLayer2ConfigExist = (
await fs.promises.readdir(
`${path.join(path.resolve(__dirname, "../../"))}/${specFolder}`
)
).includes(layer2ConfigFilename);
} catch (error) {
doesLayer2ConfigExist = false;
}
if (doesLayer2ConfigExist) {
specFile = `${specFolder}/${layer2ConfigFilename}`;
specFileName = layer2ConfigFilename;
}
else {
if (getConfig().app.mandateLayer2Config) {
const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`
logger.error(message);
return next(
new Exception(
ExceptionType.Config_AppConfig_Layer2_Missing,
message,
422
)
);
}
}
}
// Validate request body
const requestBodyKey = `${formattedVersion}-${action}-${method}-requestBody`;
logger.info(`requestBodyKey for incoming req: ${requestBodyKey}`)
Expand All @@ -228,18 +187,27 @@ export class Validator {
return res.status(400).json({ error: validateRequestBody.errors });
}
} else {
//compile schema
//Find the spec file
//load the spec file
console.log(`AGV Validation Cache miss for ${specFileName} and request body: ${requestBodyKey}`);
const apiSpecYAML = this.getApiSpec(specFile);
//parse and destructure the spec file
//call this.compileSchema(specFile)
const options = {
continueOnError: true, // Continue dereferencing despite errors
};
let dereferencedSpec: any;
dereferencedSpec = await $RefParser.dereference(apiSpecYAML, options) as OpenAPIV3.Document;

try {
await this.compileSchemas(dereferencedSpec, specFileName);
} catch (error) {
console.log('Error derefencing doc: ', error);
}
const validateRequestBody: any = this.schemaCache.get(requestBodyKey);
if (!validateRequestBody(req.body)) {
return res.status(400).json({ error: validateRequestBody.errors });
}
}

// Validate query parameters
//Validate query parameters
const queryParametersKey = `${formattedVersion}-${action}-${method}-queryParameters`;
if (this.schemaCache.has(queryParametersKey)) {
const validateQueryParameters: any = this.schemaCache.get(queryParametersKey);
Expand All @@ -249,9 +217,21 @@ export class Validator {
} else {
//compile schema
//Find the spec file
//load the spec file
console.log(`AGV Validation Cache miss for ${specFileName} and query-param-key: ${queryParametersKey}`);
const apiSpecYAML = this.getApiSpec(specFile);
//parse and destructure the spec file
//call this.compileSchema(specFile)
const options = {
continueOnError: true, // Continue dereferencing despite errors
};
let dereferencedSpec: any;
dereferencedSpec = await $RefParser.dereference(apiSpecYAML, options) as OpenAPIV3.Document;
//console.log('Dereferenced spec file: ', JSON.stringify(dereferencedSpec));

try {
await this.compileSchemas(dereferencedSpec, specFileName);
} catch (error) {
console.log('Error derefencing doc: ', error);
}
const validateRequestBody: any = this.schemaCache.get(requestBodyKey);
if (!validateRequestBody(req.body)) {
return res.status(400).json({ error: validateRequestBody.errors });
Expand All @@ -268,18 +248,28 @@ export class Validator {
} else {
//compile schema
//Find the spec file
//load the spec file
console.log(`AGV Validation Cache miss for ${specFileName} and header-key: ${headersKey}`);
const apiSpecYAML = this.getApiSpec(specFile);
//parse and destructure the spec file
//call this.compileSchema(specFile)
const options = {
continueOnError: true, // Continue dereferencing despite errors
};
let dereferencedSpec: any;
dereferencedSpec = await $RefParser.dereference(apiSpecYAML, options) as OpenAPIV3.Document;
//console.log('Dereferenced spec file: ', JSON.stringify(dereferencedSpec));

try {
await this.compileSchemas(dereferencedSpec, specFileName);
} catch (error) {
console.log('Error derefencing doc: ', error);
}
const validateRequestBody: any = this.schemaCache.get(requestBodyKey);
if (!validateRequestBody(req.body)) {
return res.status(400).json({ error: validateRequestBody.errors });
}
}

next();
};
}
}


0 comments on commit 97c51fa

Please sign in to comment.