Skip to content

Commit

Permalink
Merge pull request #179 from beckn/feat/cache-spec-on-boot
Browse files Browse the repository at this point in the history 
Added: Feature for create cache on load
  • Loading branch information
@shreyvishal
shreyvishal authored Jun 17, 2024
2 parents 76368fd + af7e86f commit 1627b60
Show file tree
Hide file tree
Showing 2 changed files with 157 additions and 52 deletions.
28 changes: 15 additions & 13 deletions src/app.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { Exception } from "./models/exception.model";
import {
BecknErrorDataType,
becknErrorSchema,
BecknErrorType,
BecknErrorType
} from "./schemas/becknError.schema";
import { RequestActions } from "./schemas/configs/actions.app.config.schema";
import { LookupCache } from "./utils/cache/lookup.cache.utils";
Expand All @@ -16,12 +16,13 @@ import { ClientUtils } from "./utils/client.utils";
import { getConfig } from "./utils/config.utils";
import { GatewayUtils } from "./utils/gateway.utils";
import logger from "./utils/logger.utils";
import { OpenApiValidatorMiddleware } from "./middlewares/schemaValidator.middleware";

const app = Express();

app.use(
Express.json({
limit: "200kb",
limit: "200kb"
})
);

Expand All @@ -35,13 +36,13 @@ const initializeExpress = async (successCallback: Function) => {
origin: "*",
optionsSuccessStatus: 200,
credentials: true,
methods: ["GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS"],
methods: ["GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS"]
})
);
app.use(
cors({
origin: "*",
methods: ["GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS"],
methods: ["GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS"]
})
);

Expand All @@ -50,10 +51,10 @@ const initializeExpress = async (successCallback: Function) => {
Express.json({
verify: (req: Request, res: Response, buf: Buffer) => {
res.locals = {
rawBody: buf.toString(),
rawBody: buf.toString()
};
},
limit: "200kb",
limit: "200kb"
})
);

Expand Down Expand Up @@ -83,24 +84,24 @@ const initializeExpress = async (successCallback: Function) => {
code: err.code,
message: err.message,
data: err.errorData,
type: BecknErrorType.domainError,
type: BecknErrorType.domainError
} as BecknErrorDataType;
res.status(err.code).json({
message: {
ack: {
status: "NACK",
},
status: "NACK"
}
},
error: errorData,
error: errorData
});
} else {
res.status(err.code || 500).json({
message: {
ack: {
status: "NACK",
},
status: "NACK"
}
},
error: err,
error: err
});
}
});
Expand All @@ -116,6 +117,7 @@ const main = async () => {
try {
await ClientUtils.initializeConnection();
await GatewayUtils.getInstance().initialize();
await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware();
if (getConfig().responseCache.enabled) {
await ResponseCache.getInstance().initialize();
}
Expand Down
181 changes: 142 additions & 39 deletions src/middlewares/schemaValidator.middleware.ts
View file
Original file line number Diff line number Diff line change
@@ -1,59 +1,158 @@
import { NextFunction, Request, Response } from "express";
import express, { NextFunction, Request, Response } from "express";
import * as OpenApiValidator from "express-openapi-validator";
import { Exception, ExceptionType } from "../models/exception.model";
import { Locals } from "../interfaces/locals.interface";
import { getConfig } from "../utils/config.utils";
import fs from "fs";
import path from "path";
import { OpenAPIV3 } from "express-openapi-validator/dist/framework/types";
import YAML from "yaml";
const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`;
import express from "express";
import { Exception, ExceptionType } from "../models/exception.model";
import { Locals } from "../interfaces/locals.interface";
import { getConfig } from "../utils/config.utils";
import logger from "../utils/logger.utils";

// Cache object
const apiSpecCache: { [filename: string]: OpenAPIV3.Document } = {};
const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`;
const specFolder = "schemas";

export class OpenApiValidatorMiddleware {
private static instance: OpenApiValidatorMiddleware;
private static cachedOpenApiValidator: {
[filename: string]: {
count: number;
requestHandler: express.RequestHandler[];
apiSpec: OpenAPIV3.Document;
};
} = {};
private static cachedFileLimit: number;

private constructor() {
OpenApiValidatorMiddleware.cachedFileLimit = 100;
}

public static getInstance(): OpenApiValidatorMiddleware {
if (!OpenApiValidatorMiddleware.instance) {
OpenApiValidatorMiddleware.instance = new OpenApiValidatorMiddleware();
}
return OpenApiValidatorMiddleware.instance;
}

// Function to load and cache the API spec
const loadApiSpec = (specFile: string): OpenAPIV3.Document => {
if (!apiSpecCache[specFile]) {
logger.info(`Cache Not found loadApiSpec file. Loading.... ${specFile}`);
private getApiSpec(specFile: string): OpenAPIV3.Document {
const apiSpecYAML = fs.readFileSync(specFile, "utf8");
const apiSpec = YAML.parse(apiSpecYAML);
apiSpecCache[specFile] = apiSpec;
return apiSpec;
}
return apiSpecCache[specFile];
};

let cachedOpenApiValidator: express.RequestHandler[] | null = null;
let cachedSpecFile: string | null = null;
public initOpenApiMiddleware(): void {
try {
const files = fs.readdirSync(specFolder);
const fileNames = files.filter(
(file) =>
fs.lstatSync(path.join(specFolder, file)).isFile() &&
(file.endsWith(".yaml") || file.endsWith(".yml"))
);
const cachedFileLimit: number =
OpenApiValidatorMiddleware.cachedFileLimit;
logger.info(`OpenAPIValidator Cache count ${cachedFileLimit}`);
for (let i = 0; i < cachedFileLimit && fileNames[i]; i++) {
const file = `${specFolder}/${fileNames[i]}`;
if (!OpenApiValidatorMiddleware.cachedOpenApiValidator[file]) {
logger.info(
`Intially cache Not found loadApiSpec file. Loading.... ${file}`
);
const apiSpec = this.getApiSpec(file);
OpenApiValidatorMiddleware.cachedOpenApiValidator[file] = {
apiSpec,
count: 0,
requestHandler: OpenApiValidator.middleware({
apiSpec,
validateRequests: true,
validateResponses: false,
$refParser: {
mode: "dereference"
}
})
};
}
}
} catch (err) {
logger.error("Error in initializing open API middleware", err);
}
}

// Function to initialize and cache the OpenAPI validator middleware
const getOpenApiValidatorMiddleware = (specFile: string) => {
if (!cachedOpenApiValidator || cachedSpecFile !== specFile) {
logger.info(
`Cache Not found for OpenApiValidator middleware. Loading.... ${specFile}`
);
const apiSpec = loadApiSpec(specFile);
cachedOpenApiValidator = OpenApiValidator.middleware({
apiSpec,
validateRequests: true,
validateResponses: false,
$refParser: {
mode: "dereference"
public getOpenApiMiddleware(specFile: string): express.RequestHandler[] {
try {
let requestHandler: express.RequestHandler[];
if (OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]) {
const cachedValidator =
OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile];
cachedValidator.count =
cachedValidator.count > 1000
? cachedValidator.count
: cachedValidator.count + 1;
logger.info(`Cache found for spec ${specFile}`);
requestHandler = cachedValidator.requestHandler;
} else {
const cashedSpec = Object.entries(
OpenApiValidatorMiddleware.cachedOpenApiValidator
);
const cachedFileLimit: number =
OpenApiValidatorMiddleware.cachedFileLimit;
if (cashedSpec.length >= cachedFileLimit) {
const specWithLeastCount =
cashedSpec.reduce((minEntry, currentEntry) => {
return currentEntry[1].count < minEntry[1].count
? currentEntry
: minEntry;
}) || cashedSpec[0];
logger.info(
`Cache count reached limit. Deleting from cache.... ${specWithLeastCount[0]}`
);
delete OpenApiValidatorMiddleware.cachedOpenApiValidator[
specWithLeastCount[0]
];
}
logger.info(
`Cache Not found loadApiSpec file. Loading.... ${specFile}`
);
const apiSpec = this.getApiSpec(specFile);
OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile] = {
apiSpec,
count: 1,
requestHandler: OpenApiValidator.middleware({
apiSpec,
validateRequests: true,
validateResponses: false,
$refParser: {
mode: "dereference"
}
})
};
requestHandler =
OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]
.requestHandler;
}
});
cachedSpecFile = specFile;
const cacheStats = Object.entries(
OpenApiValidatorMiddleware.cachedOpenApiValidator
).map((cache) => {
return {
count: cache[1].count,
specFile: cache[0]
};
});
console.table(cacheStats);
return requestHandler;
} catch (err) {
logger.error("Error in getOpenApiMiddleware", err);
return [];
}
}
return cachedOpenApiValidator;
};
}

export const schemaErrorHandler = (
err: any,
req: Request,
res: Response,
next: NextFunction
) => {
logger.error("OpenApiValidator Error", err);
if (err instanceof Exception) {
next(err);
} else {
Expand All @@ -76,7 +175,7 @@ export const openApiValidatorMiddleware = async (
const version = req?.body?.context?.core_version
? req?.body?.context?.core_version
: req?.body?.context?.version;
let specFile = `schemas/core_${version}.yaml`;
let specFile = `${specFolder}/core_${version}.yaml`;

if (getConfig().app.useLayer2Config) {
let doesLayer2ConfigExist = false;
Expand All @@ -86,27 +185,31 @@ export const openApiValidatorMiddleware = async (
try {
doesLayer2ConfigExist = (
await fs.promises.readdir(
`${path.join(path.resolve(__dirname, "../../"))}/schemas`
`${path.join(path.resolve(__dirname, "../../"))}/${specFolder}`
)
).includes(layer2ConfigFilename);
} catch (error) {
doesLayer2ConfigExist = false;
}
if (doesLayer2ConfigExist) specFile = `schemas/${layer2ConfigFilename}`;
if (doesLayer2ConfigExist)
specFile = `${specFolder}/${layer2ConfigFilename}`;
else {
if (getConfig().app.mandateLayer2Config) {
const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`;
logger.error(message);
return next(
new Exception(
ExceptionType.Config_AppConfig_Layer2_Missing,
`Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`,
message,
422
)
);
}
}
}

const openApiValidator = getOpenApiValidatorMiddleware(specFile);
const openApiValidator =
OpenApiValidatorMiddleware.getInstance().getOpenApiMiddleware(specFile);

const walkSubstack = function (
stack: any,
Expand Down

0 comments on commit 1627b60

Please sign in to comment.