diff --git a/package.json b/package.json index db22d37..677144e 100644 --- a/package.json +++ b/package.json @@ -33,14 +33,15 @@ "dotenv": "^16.4.1", "express": "^4.18.1", "express-openapi-validator": "^5.1.6", - "yaml": "^2.4.2", "ioredis": "^5.0.6", "libsodium-wrappers": "^0.7.9", "mongodb": "^4.7.0", + "node-mocks-http": "^1.14.1", "request-ip": "^3.3.0", "uuid": "^8.3.2", "winston": "^3.7.2", "winston-daily-rotate-file": "^4.7.1", + "yaml": "^2.4.2", "zod": "^3.14.2" } } diff --git a/src/app.ts b/src/app.ts index d5b2752..e14903d 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,7 +4,7 @@ import { Exception } from "./models/exception.model"; import { BecknErrorDataType, becknErrorSchema, - BecknErrorType, + BecknErrorType } from "./schemas/becknError.schema"; import { RequestActions } from "./schemas/configs/actions.app.config.schema"; import { LookupCache } from "./utils/cache/lookup.cache.utils"; @@ -16,12 +16,13 @@ import { ClientUtils } from "./utils/client.utils"; import { getConfig } from "./utils/config.utils"; import { GatewayUtils } from "./utils/gateway.utils"; import logger from "./utils/logger.utils"; +import { OpenApiValidatorMiddleware } from "./middlewares/schemaValidator.middleware"; const app = Express(); app.use( Express.json({ - limit: "200kb", + limit: "200kb" }) ); @@ -35,13 +36,13 @@ const initializeExpress = async (successCallback: Function) => { origin: "*", optionsSuccessStatus: 200, credentials: true, - methods: ["GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS"], + methods: ["GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS"] }) ); app.use( cors({ origin: "*", - methods: ["GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS"], + methods: ["GET", "PUT", "POST", "PATCH", "DELETE", "OPTIONS"] }) ); @@ -50,10 +51,10 @@ const initializeExpress = async (successCallback: Function) => { Express.json({ verify: (req: Request, res: Response, buf: Buffer) => { res.locals = { - rawBody: buf.toString(), + rawBody: buf.toString() }; }, - limit: "200kb", + limit: "200kb" }) ); @@ -83,24 +84,24 @@ const initializeExpress = async (successCallback: Function) => { code: err.code, message: err.message, data: err.errorData, - type: BecknErrorType.domainError, + type: BecknErrorType.domainError } as BecknErrorDataType; res.status(err.code).json({ message: { ack: { - status: "NACK", - }, + status: "NACK" + } }, - error: errorData, + error: errorData }); } else { res.status(err.code || 500).json({ message: { ack: { - status: "NACK", - }, + status: "NACK" + } }, - error: err, + error: err }); } }); @@ -131,6 +132,7 @@ const main = async () => { getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) ); }); + await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware(); } catch (err) { if (err instanceof Exception) { logger.error(err.toString()); diff --git a/src/middlewares/schemaValidator.middleware.ts b/src/middlewares/schemaValidator.middleware.ts index e0fdab8..667730b 100644 --- a/src/middlewares/schemaValidator.middleware.ts +++ b/src/middlewares/schemaValidator.middleware.ts @@ -1,51 +1,235 @@ -import { NextFunction, Request, Response } from "express"; +import express, { NextFunction, Request, Response } from "express"; import * as OpenApiValidator from "express-openapi-validator"; -import { Exception, ExceptionType } from "../models/exception.model"; -import { Locals } from "../interfaces/locals.interface"; -import { getConfig } from "../utils/config.utils"; import fs from "fs"; import path from "path"; import { OpenAPIV3 } from "express-openapi-validator/dist/framework/types"; import YAML from "yaml"; -const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`; -import express from "express"; +import { Exception, ExceptionType } from "../models/exception.model"; +import { Locals } from "../interfaces/locals.interface"; +import { getConfig } from "../utils/config.utils"; import logger from "../utils/logger.utils"; +import { + RequestActions, + ResponseActions +} from "../schemas/configs/actions.app.config.schema"; +import * as httpMocks from "node-mocks-http"; +import { v4 as uuid_v4 } from "uuid"; +import { AppMode } from "../schemas/configs/app.config.schema"; +import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; +const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`; +const specFolder = "schemas"; + +export class OpenApiValidatorMiddleware { + private static instance: OpenApiValidatorMiddleware; + private static cachedOpenApiValidator: { + [filename: string]: { + count: number; + requestHandler: express.RequestHandler[]; + apiSpec: OpenAPIV3.Document; + }; + } = {}; + private static cachedFileLimit: number; -// Cache object -const apiSpecCache: { [filename: string]: OpenAPIV3.Document } = {}; + private constructor() { + OpenApiValidatorMiddleware.cachedFileLimit = 100; + } -// Function to load and cache the API spec -const loadApiSpec = (specFile: string): OpenAPIV3.Document => { - if (!apiSpecCache[specFile]) { - logger.info(`Cache Not found loadApiSpec file. Loading.... ${specFile}`); + public static getInstance(): OpenApiValidatorMiddleware { + if (!OpenApiValidatorMiddleware.instance) { + OpenApiValidatorMiddleware.instance = new OpenApiValidatorMiddleware(); + } + return OpenApiValidatorMiddleware.instance; + } + + private getApiSpec(specFile: string): OpenAPIV3.Document { const apiSpecYAML = fs.readFileSync(specFile, "utf8"); const apiSpec = YAML.parse(apiSpecYAML); - apiSpecCache[specFile] = apiSpec; + return apiSpec; } - return apiSpecCache[specFile]; -}; -let cachedOpenApiValidator: express.RequestHandler[] | null = null; -let cachedSpecFile: string | null = null; + public async initOpenApiMiddleware() { + try { + const files = fs.readdirSync(specFolder); + const fileNames = files.filter( + (file) => + fs.lstatSync(path.join(specFolder, file)).isFile() && + (file.endsWith(".yaml") || file.endsWith(".yml")) + ); + const cachedFileLimit: number = + OpenApiValidatorMiddleware.cachedFileLimit; + logger.info(`OpenAPIValidator Cache count ${cachedFileLimit}`); + for (let i = 0; i < cachedFileLimit && fileNames[i]; i++) { + const file = `${specFolder}/${fileNames[i]}`; + if (!OpenApiValidatorMiddleware.cachedOpenApiValidator[file]) { + logger.info( + `Intially cache Not found loadApiSpec file. Loading.... ${file}` + ); + const apiSpec = this.getApiSpec(file); + const requestHandler = OpenApiValidator.middleware({ + apiSpec, + validateRequests: true, + validateResponses: false, + $refParser: { + mode: "dereference" + } + }); + OpenApiValidatorMiddleware.cachedOpenApiValidator[file] = { + apiSpec, + count: 0, + requestHandler: requestHandler + }; + await initializeOpenApiValidatorCache(requestHandler, file); + } + } + } catch (err) { + logger.error("Error in initializing open API middleware", err); + } + } -// Function to initialize and cache the OpenAPI validator middleware -const getOpenApiValidatorMiddleware = (specFile: string) => { - if (!cachedOpenApiValidator || cachedSpecFile !== specFile) { - logger.info( - `Cache Not found for OpenApiValidator middleware. Loading.... ${specFile}` - ); - const apiSpec = loadApiSpec(specFile); - cachedOpenApiValidator = OpenApiValidator.middleware({ - apiSpec, - validateRequests: true, - validateResponses: false, - $refParser: { - mode: "dereference" + public getOpenApiMiddleware(specFile: string): express.RequestHandler[] { + try { + let requestHandler: express.RequestHandler[]; + if (OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]) { + const cachedValidator = + OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]; + cachedValidator.count = + cachedValidator.count > 1000 + ? cachedValidator.count + : cachedValidator.count + 1; + logger.info(`Cache found for spec ${specFile}`); + requestHandler = cachedValidator.requestHandler; + } else { + const cashedSpec = Object.entries( + OpenApiValidatorMiddleware.cachedOpenApiValidator + ); + const cachedFileLimit: number = + OpenApiValidatorMiddleware.cachedFileLimit; + if (cashedSpec.length >= cachedFileLimit) { + const specWithLeastCount = + cashedSpec.reduce((minEntry, currentEntry) => { + return currentEntry[1].count < minEntry[1].count + ? currentEntry + : minEntry; + }) || cashedSpec[0]; + logger.info( + `Cache count reached limit. Deleting from cache.... ${specWithLeastCount[0]}` + ); + delete OpenApiValidatorMiddleware.cachedOpenApiValidator[ + specWithLeastCount[0] + ]; + } + logger.info( + `Cache Not found loadApiSpec file. Loading.... ${specFile}` + ); + const apiSpec = this.getApiSpec(specFile); + OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile] = { + apiSpec, + count: 1, + requestHandler: OpenApiValidator.middleware({ + apiSpec, + validateRequests: true, + validateResponses: false, + $refParser: { + mode: "dereference" + } + }) + }; + requestHandler = + OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile] + .requestHandler; } + + const cacheStats = Object.entries( + OpenApiValidatorMiddleware.cachedOpenApiValidator + ).map((cache) => { + return { + count: cache[1].count, + specFile: cache[0] + }; + }); + console.table(cacheStats); + return requestHandler; + } catch (err) { + logger.error("Error in getOpenApiMiddleware", err); + return []; + } + } +} + +const initializeOpenApiValidatorCache = async ( + stack: any, + specFile: string +) => { + try { + let actions: string[] = []; + if ( + (getConfig().app.mode === AppMode.bap && + getConfig().app.gateway.mode === GatewayMode.client) || + (getConfig().app.mode === AppMode.bpp && + getConfig().app.gateway.mode === GatewayMode.network) + ) { + actions = Object.keys(RequestActions); + } else { + actions = Object.keys(ResponseActions); + } + + actions.forEach((action) => { + const mockRequest = (body: any) => { + const req = httpMocks.createRequest({ + method: "POST", + url: `/${action}`, + headers: { + "Content-Type": "application/json", + Authorization: uuid_v4() + }, + body: body + }); + + req.app = { + enabled: (setting: any) => { + if ( + setting === "strict routing" || + setting === "case sensitive routing" + ) { + return true; + } + return false; + } + } as any; + return req; + }; + + const reqObj = mockRequest({ + context: { action: `${action}` }, + message: {} + }); + + walkSubstack(stack, reqObj, {}, () => { + return; + }); }); - cachedSpecFile = specFile; + } catch (error: any) {} +}; + +const walkSubstack = function ( + stack: any, + req: any, + res: any, + next: NextFunction +) { + if (typeof stack === "function") { + stack = [stack]; } - return cachedOpenApiValidator; + const walkStack = function (i: any, err?: any) { + if (err) { + return schemaErrorHandler(err, req, res, next); + } + if (i >= stack.length) { + return next(); + } + stack[i](req, res, walkStack.bind(null, i + 1)); + }; + walkStack(0); }; export const schemaErrorHandler = ( @@ -76,7 +260,7 @@ export const openApiValidatorMiddleware = async ( const version = req?.body?.context?.core_version ? req?.body?.context?.core_version : req?.body?.context?.version; - let specFile = `schemas/core_${version}.yaml`; + let specFile = `${specFolder}/core_${version}.yaml`; if (getConfig().app.useLayer2Config) { let doesLayer2ConfigExist = false; @@ -86,47 +270,29 @@ export const openApiValidatorMiddleware = async ( try { doesLayer2ConfigExist = ( await fs.promises.readdir( - `${path.join(path.resolve(__dirname, "../../"))}/schemas` + `${path.join(path.resolve(__dirname, "../../"))}/${specFolder}` ) ).includes(layer2ConfigFilename); } catch (error) { doesLayer2ConfigExist = false; } - if (doesLayer2ConfigExist) specFile = `schemas/${layer2ConfigFilename}`; + if (doesLayer2ConfigExist) + specFile = `${specFolder}/${layer2ConfigFilename}`; else { if (getConfig().app.mandateLayer2Config) { + const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`; + logger.error(message); return next( new Exception( ExceptionType.Config_AppConfig_Layer2_Missing, - `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`, + message, 422 ) ); } } } - - const openApiValidator = getOpenApiValidatorMiddleware(specFile); - - const walkSubstack = function ( - stack: any, - req: any, - res: any, - next: NextFunction - ) { - if (typeof stack === "function") { - stack = [stack]; - } - const walkStack = function (i: any, err?: any) { - if (err) { - return schemaErrorHandler(err, req, res, next); - } - if (i >= stack.length) { - return next(); - } - stack[i](req, res, walkStack.bind(null, i + 1)); - }; - walkStack(0); - }; + const openApiValidator = + OpenApiValidatorMiddleware.getInstance().getOpenApiMiddleware(specFile); walkSubstack([...openApiValidator], req, res, next); }; diff --git a/src/utils/mongo.utils.ts b/src/utils/mongo.utils.ts index 0c4b4f3..8137f0d 100644 --- a/src/utils/mongo.utils.ts +++ b/src/utils/mongo.utils.ts @@ -2,41 +2,50 @@ import { Db, MongoClient } from "mongodb"; import { Exception, ExceptionType } from "../models/exception.model"; import logger from "./logger.utils"; -export class DBClient{ - private db: Db; - private client: MongoClient; - public isConnected: boolean = false; - - constructor(dbURL: string){ - this.client = new MongoClient(dbURL, { - minPoolSize: 10, - maxPoolSize: 15, - }); - - this.db = this.client.db(); +export class DBClient { + private db: Db; + private client: MongoClient; + public isConnected: boolean = false; + + constructor(dbURL: string) { + this.client = new MongoClient(dbURL, { + minPoolSize: 10, + maxPoolSize: 15 + }); + + this.db = this.client.db(); + } + + public async connect(): Promise { + try { + this.client = await this.client.connect(); + this.db = this.client.db(); + this.isConnected = true; + logger.info(`Mongo Client Connected For DB: ${this.db.databaseName}`); + } catch (error: any) {} + } + + public getDB(): Db { + if (!this.isConnected) { + throw new Exception( + ExceptionType.Mongo_ClientNotInitialized, + "Mongo client is not connected.", + 500 + ); } - public async connect(): Promise { - this.client = await this.client.connect(); - this.db = this.client.db(); - this.isConnected = true; - logger.info(`Mongo Client Connected For DB: ${this.db.databaseName}`); - } - - public getDB(): Db{ - if(!this.isConnected){ - throw new Exception(ExceptionType.Mongo_ClientNotInitialized, "Mongo client is not connected.", 500); - } - - return this.db; - } - - public getClient(): MongoClient{ - if(!this.isConnected){ - throw new Exception(ExceptionType.Mongo_ClientNotInitialized, "Mongo client is not connected.", 500); - } + return this.db; + } - return this.client; + public getClient(): MongoClient { + if (!this.isConnected) { + throw new Exception( + ExceptionType.Mongo_ClientNotInitialized, + "Mongo client is not connected.", + 500 + ); } -} \ No newline at end of file + return this.client; + } +}