From 62fa6839624d8533dc930620ea9dd13d4e7423c4 Mon Sep 17 00:00:00 2001 From: Abhishek Y Date: Sat, 29 Jun 2024 12:53:38 +0530 Subject: [PATCH] Implemented logic to load OpenAPI validator spec at app load Implemented logic to load OpenAPI validator spec at app load to improve latency issue Removed unused import from some files Added /status routs to debug version of app deployed Added log for OpenAPI validator failure Added log for layer2 config message Added action in latency logs --- config/config-sample-client-localhost.yaml | 7 + config/config-sample-network-localhost.yaml | 7 + config/config-sample.yaml | 7 + config/new-config-sample.yaml | 7 + config/samples/bap-client.yaml | 7 + config/samples/bap-network.yaml | 7 + config/samples/bpp-client.yaml | 7 + config/samples/bpp-network.yaml | 7 + package.json | 4 +- src/app.ts | 35 ++- src/controllers/bap.response.controller.ts | 19 +- src/controllers/bap.trigger.controller.ts | 7 +- src/controllers/bpp.request.controller.ts | 8 +- src/controllers/bpp.response.controller.ts | 9 +- src/middlewares/schemaValidator.middleware.ts | 270 ++++++++++++++---- src/routes/requests.routes.ts | 27 +- src/routes/responses.routes.ts | 18 +- src/schemas/configs/app.config.schema.ts | 4 + src/utils/becknRequester.utils.ts | 4 +- src/utils/callback.utils.ts | 2 +- src/utils/syncResponses.utils.ts | 2 +- 21 files changed, 334 insertions(+), 131 deletions(-) diff --git a/config/config-sample-client-localhost.yaml b/config/config-sample-client-localhost.yaml index 369a1bf..0ece75a 100644 --- a/config/config-sample-client-localhost.yaml +++ b/config/config-sample-client-localhost.yaml @@ -114,3 +114,10 @@ app: useHMACForWebhook: false sharedKeyForWebhookHMAC: "" + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 + initialFilesToCache: "" diff --git a/config/config-sample-network-localhost.yaml b/config/config-sample-network-localhost.yaml index b9280bc..8109dd5 100644 --- a/config/config-sample-network-localhost.yaml +++ b/config/config-sample-network-localhost.yaml @@ -113,3 +113,10 @@ app: useHMACForWebhook: false sharedKeyForWebhookHMAC: "" + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 + initialFilesToCache: "" diff --git a/config/config-sample.yaml b/config/config-sample.yaml index 80b0771..99ced61 100644 --- a/config/config-sample.yaml +++ b/config/config-sample.yaml @@ -93,3 +93,10 @@ app: useHMACForWebhook: false sharedKeyForWebhookHMAC: "" + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 + initialFilesToCache: "" diff --git a/config/new-config-sample.yaml b/config/new-config-sample.yaml index f8df638..5c76221 100644 --- a/config/new-config-sample.yaml +++ b/config/new-config-sample.yaml @@ -94,3 +94,10 @@ app: useHMACForWebhook: false sharedKeyForWebhookHMAC: "" + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 + initialFilesToCache: "" diff --git a/config/samples/bap-client.yaml b/config/samples/bap-client.yaml index fad3e0f..0f029df 100644 --- a/config/samples/bap-client.yaml +++ b/config/samples/bap-client.yaml @@ -134,3 +134,10 @@ app: useHMACForWebhook: false sharedKeyForWebhookHMAC: "" + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 + initialFilesToCache: "" diff --git a/config/samples/bap-network.yaml b/config/samples/bap-network.yaml index 0470861..b4c269d 100644 --- a/config/samples/bap-network.yaml +++ b/config/samples/bap-network.yaml @@ -134,3 +134,10 @@ app: useHMACForWebhook: false sharedKeyForWebhookHMAC: "" + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 + initialFilesToCache: "" diff --git a/config/samples/bpp-client.yaml b/config/samples/bpp-client.yaml index a08cd70..b480ea4 100644 --- a/config/samples/bpp-client.yaml +++ b/config/samples/bpp-client.yaml @@ -132,3 +132,10 @@ app: useHMACForWebhook: false sharedKeyForWebhookHMAC: "" + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 + initialFilesToCache: "" diff --git a/config/samples/bpp-network.yaml b/config/samples/bpp-network.yaml index 32d79b9..e87bb52 100644 --- a/config/samples/bpp-network.yaml +++ b/config/samples/bpp-network.yaml @@ -132,3 +132,10 @@ app: useHMACForWebhook: false sharedKeyForWebhookHMAC: "" + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 + initialFilesToCache: "" diff --git a/package.json b/package.json index db22d37..f605082 100644 --- a/package.json +++ b/package.json @@ -33,14 +33,16 @@ "dotenv": "^16.4.1", "express": "^4.18.1", "express-openapi-validator": "^5.1.6", - "yaml": "^2.4.2", + "express-status-monitor": "^1.3.4", "ioredis": "^5.0.6", "libsodium-wrappers": "^0.7.9", "mongodb": "^4.7.0", + "node-mocks-http": "^1.15.0", "request-ip": "^3.3.0", "uuid": "^8.3.2", "winston": "^3.7.2", "winston-daily-rotate-file": "^4.7.1", + "yaml": "^2.4.2", "zod": "^3.14.2" } } diff --git a/src/app.ts b/src/app.ts index d5b2752..b423f30 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,19 +3,16 @@ import cors from "cors"; import { Exception } from "./models/exception.model"; import { BecknErrorDataType, - becknErrorSchema, BecknErrorType, } from "./schemas/becknError.schema"; -import { RequestActions } from "./schemas/configs/actions.app.config.schema"; import { LookupCache } from "./utils/cache/lookup.cache.utils"; import { RequestCache } from "./utils/cache/request.cache.utils"; import { ResponseCache } from "./utils/cache/response.cache.utils"; -import { SyncCache } from "./utils/cache/sync.cache.utils"; import { ClientUtils } from "./utils/client.utils"; - import { getConfig } from "./utils/config.utils"; import { GatewayUtils } from "./utils/gateway.utils"; import logger from "./utils/logger.utils"; +import { OpenApiValidatorMiddleware } from "./middlewares/schemaValidator.middleware"; const app = Express(); @@ -25,8 +22,16 @@ app.use( }) ); -const initializeExpress = async (successCallback: Function) => { +const initializeExpress = async () => { const app = Express(); + app.use( + require("express-status-monitor")({ + path: "/process" + }) + ); + app.get("/status", async (req: Request, res: Response, next: NextFunction) => { + res.status(200).send('Added logic to cache OpenAPI validator spec on app load new'); + }); // Enabling Cors app.options( @@ -108,7 +113,6 @@ const initializeExpress = async (successCallback: Function) => { const PORT: number = getConfig().server.port; app.listen(PORT, () => { logger.info("Protocol Server started on PORT : " + PORT); - successCallback(); }); }; @@ -122,15 +126,16 @@ const main = async () => { await LookupCache.getInstance().initialize(); await RequestCache.getInstance().initialize(); - await initializeExpress(() => { - logger.info("Protocol Server Started Successfully"); - logger.info("Mode: " + getConfig().app.mode.toLocaleUpperCase()); - logger.info( - "Gateway Type: " + - getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) + - getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) - ); - }); + await initializeExpress(); + logger.info("Protocol Server Started Successfully"); + logger.info("Mode: " + getConfig().app.mode.toLocaleUpperCase()); + logger.info( + "Gateway Type: " + + getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) + + getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) + ); + await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware(); + logger.info('Initialized openapi validator middleware'); } catch (err) { if (err instanceof Exception) { logger.error(err.toString()); diff --git a/src/controllers/bap.response.controller.ts b/src/controllers/bap.response.controller.ts index 5bb8a0a..5ba5e10 100644 --- a/src/controllers/bap.response.controller.ts +++ b/src/controllers/bap.response.controller.ts @@ -1,8 +1,9 @@ import { NextFunction, Request, Response } from "express"; +import * as AmqbLib from "amqplib"; +import moment from "moment"; import { Locals } from "../interfaces/locals.interface"; import { ResponseActions } from "../schemas/configs/actions.app.config.schema"; import logger from "../utils/logger.utils"; -import * as AmqbLib from "amqplib"; import { Exception, ExceptionType } from "../models/exception.model"; import { ActionUtils } from "../utils/actions.utils"; import { RequestCache } from "../utils/cache/request.cache.utils"; @@ -21,7 +22,6 @@ import { createTelemetryEvent, processTelemetry, } from "../utils/telemetry.utils"; -import moment from "moment"; export const bapNetworkResponseHandler = async ( req: Request, @@ -67,7 +67,8 @@ export const bapNetworkResponseHandler = async ( await GatewayUtils.getInstance().sendToClientSideGateway(req.body); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode + } REV EXIT: ${new Date().valueOf()}` ); } catch (err) { let exception: Exception | null = null; @@ -94,19 +95,17 @@ export const bapNetworkResponseSettler = async ( logger.info( "Protocol Client Server (Network Settler) recieving message from inbox queue" ); - - const responseBody = JSON.parse(message?.content.toString()!); - + let responseBody = JSON.parse(message?.content.toString()!); logger.info( `Response from BPP NETWORK:\n ${JSON.stringify(responseBody)}\n\n` ); - const message_id = responseBody.context.message_id; const action = ActionUtils.getCorrespondingRequestAction( responseBody.context.action ); console.log( - `TMTR - ${message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` + `TMTR - ${message_id} - ${action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode + } REV ENTRY: ${new Date().valueOf()}` ); const unsolicitedWebhookUrl = getConfig().app.unsolicitedWebhook?.url; const requestCache = await RequestCache.getInstance().check( @@ -114,6 +113,10 @@ export const bapNetworkResponseSettler = async ( action ); if (!requestCache && unsolicitedWebhookUrl) { + responseBody = { + context: responseBody.context, + responses: [responseBody] + }; unsolicitedCallback(responseBody); return; } diff --git a/src/controllers/bap.trigger.controller.ts b/src/controllers/bap.trigger.controller.ts index 14cda63..b8a5586 100644 --- a/src/controllers/bap.trigger.controller.ts +++ b/src/controllers/bap.trigger.controller.ts @@ -1,8 +1,8 @@ import { NextFunction, Request, Response } from "express"; +import * as AmqbLib from "amqplib"; import { Locals } from "../interfaces/locals.interface"; import { RequestActions } from "../schemas/configs/actions.app.config.schema"; import logger from "../utils/logger.utils"; -import * as AmqbLib from "amqplib"; import { acknowledgeACK, acknowledgeNACK @@ -27,6 +27,7 @@ import { createTelemetryEvent, processTelemetry } from "../utils/telemetry.utils"; + const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`; export const bapClientTriggerHandler = async ( @@ -73,7 +74,7 @@ export const bapClientTriggerHandler = async ( logger.info(`Request from client:\n ${JSON.stringify(req.body)}\n`); await GatewayUtils.getInstance().sendToNetworkSideGateway(req.body); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` ); if (getConfig().client.type == ClientConfigType.synchronous) { sendSyncResponses( @@ -107,7 +108,7 @@ export const bapClientTriggerSettler = async ( try { const body = (JSON.parse(message?.content.toString()!) as any) console.log( - `TMTR - ${body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` + `TMTR - ${body?.context?.message_id} - ${body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` ); logger.info( "Protocol Network Server (Client Settler) recieving message from outbox queue" diff --git a/src/controllers/bpp.request.controller.ts b/src/controllers/bpp.request.controller.ts index 1a3626f..af7a114 100644 --- a/src/controllers/bpp.request.controller.ts +++ b/src/controllers/bpp.request.controller.ts @@ -1,14 +1,14 @@ import { NextFunction, Request, Response } from "express"; +import * as AmqbLib from "amqplib"; +import moment from "moment"; import { RequestActions } from "../schemas/configs/actions.app.config.schema"; import logger from "../utils/logger.utils"; -import * as AmqbLib from "amqplib"; import { Exception, ExceptionType } from "../models/exception.model"; import { acknowledgeACK } from "../utils/acknowledgement.utils"; import { GatewayUtils } from "../utils/gateway.utils"; import { RequestCache } from "../utils/cache/request.cache.utils"; import { parseRequestCache } from "../schemas/cache/request.cache.schema"; import { Locals } from "../interfaces/locals.interface"; -import moment from "moment"; import { getConfig } from "../utils/config.utils"; import { ClientConfigType } from "../schemas/configs/client.config.schema"; import { requestCallback } from "../utils/callback.utils"; @@ -46,7 +46,7 @@ export const bppNetworkRequestHandler = async ( await processTelemetry(); } console.log( - `TMTR - ${req.body.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` + `TMTR - ${req.body.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` ); await GatewayUtils.getInstance().sendToClientSideGateway(req.body); } catch (err) { @@ -72,7 +72,7 @@ export const bppNetworkRequestSettler = async ( try { const requestBody = JSON.parse(msg?.content.toString()!); console.log( - `TMTR - ${requestBody?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` + `TMTR - ${requestBody?.context?.message_id} - ${requestBody?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` ); // Generate Telemetry if enabled if (getConfig().app.telemetry.enabled && getConfig().app.telemetry.url) { diff --git a/src/controllers/bpp.response.controller.ts b/src/controllers/bpp.response.controller.ts index 473a550..377901c 100644 --- a/src/controllers/bpp.response.controller.ts +++ b/src/controllers/bpp.response.controller.ts @@ -1,11 +1,11 @@ import { Request, Response, NextFunction } from "express"; +import * as AmqbLib from "amqplib"; +import moment from "moment"; import { Locals } from "../interfaces/locals.interface"; import { - RequestActions, ResponseActions } from "../schemas/configs/actions.app.config.schema"; import logger from "../utils/logger.utils"; -import * as AmqbLib from "amqplib"; import { Exception, ExceptionType } from "../models/exception.model"; import { GatewayUtils } from "../utils/gateway.utils"; import { RequestCache } from "../utils/cache/request.cache.utils"; @@ -28,7 +28,6 @@ import { processTelemetry } from "../utils/telemetry.utils"; import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; -import moment from "moment"; import { getSubscriberDetails } from "../utils/lookup.utils"; export const bppClientResponseHandler = async ( @@ -41,7 +40,7 @@ export const bppClientResponseHandler = async ( acknowledgeACK(res, req.body.context); await GatewayUtils.getInstance().sendToNetworkSideGateway(req.body); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` ); } catch (err) { let exception: Exception | null = null; @@ -68,7 +67,7 @@ export const bppClientResponseSettler = async ( const context = JSON.parse(JSON.stringify(responseBody.context)); const message_id = responseBody.context.message_id; console.log( - `TMTR - ${context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` + `TMTR - ${context?.message_id} - ${context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` ); const requestAction = ActionUtils.getCorrespondingRequestAction( responseBody.context.action diff --git a/src/middlewares/schemaValidator.middleware.ts b/src/middlewares/schemaValidator.middleware.ts index e0fdab8..c6b8657 100644 --- a/src/middlewares/schemaValidator.middleware.ts +++ b/src/middlewares/schemaValidator.middleware.ts @@ -1,51 +1,196 @@ -import { NextFunction, Request, Response } from "express"; +import express, { NextFunction, Request, Response } from "express"; import * as OpenApiValidator from "express-openapi-validator"; +import fs from "fs"; +import path from "path"; +import YAML from "yaml"; +import * as httpMocks from "node-mocks-http"; +import { v4 as uuid_v4 } from "uuid"; import { Exception, ExceptionType } from "../models/exception.model"; import { Locals } from "../interfaces/locals.interface"; import { getConfig } from "../utils/config.utils"; -import fs from "fs"; -import path from "path"; import { OpenAPIV3 } from "express-openapi-validator/dist/framework/types"; -import YAML from "yaml"; -const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`; -import express from "express"; import logger from "../utils/logger.utils"; +import { AppMode } from "../schemas/configs/app.config.schema"; +import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; +import { + RequestActions, + ResponseActions +} from "../schemas/configs/actions.app.config.schema"; + +const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`; +const specFolder = 'schemas'; + +export class OpenApiValidatorMiddleware { + private static instance: OpenApiValidatorMiddleware; + private static cachedOpenApiValidator: { + [filename: string]: { + count: number, + requestHandler: express.RequestHandler[], + apiSpec: OpenAPIV3.Document + } + } = {}; + private static cachedFileLimit: number; + + private constructor() { + OpenApiValidatorMiddleware.cachedFileLimit = getConfig().app.openAPIValidator?.cachedFileLimit || 20; + } -// Cache object -const apiSpecCache: { [filename: string]: OpenAPIV3.Document } = {}; + public static getInstance(): OpenApiValidatorMiddleware { + if (!OpenApiValidatorMiddleware.instance) { + OpenApiValidatorMiddleware.instance = new OpenApiValidatorMiddleware(); + } + return OpenApiValidatorMiddleware.instance; + } -// Function to load and cache the API spec -const loadApiSpec = (specFile: string): OpenAPIV3.Document => { - if (!apiSpecCache[specFile]) { - logger.info(`Cache Not found loadApiSpec file. Loading.... ${specFile}`); + private getApiSpec(specFile: string): OpenAPIV3.Document { const apiSpecYAML = fs.readFileSync(specFile, "utf8"); const apiSpec = YAML.parse(apiSpecYAML); - apiSpecCache[specFile] = apiSpec; - } - return apiSpecCache[specFile]; -}; + return apiSpec; + }; -let cachedOpenApiValidator: express.RequestHandler[] | null = null; -let cachedSpecFile: string | null = null; + public async initOpenApiMiddleware(): Promise { + try { + let fileToCache = getConfig().app?.openAPIValidator?.initialFilesToCache; + let fileNames, noOfFileToCache = 0; + const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit; + logger.info(`OpenAPIValidator Total Cache capacity ${cachedFileLimit}`); + if (fileToCache) { + fileNames = fileToCache.split(/\s*,\s*/).map(item => item.trim()); + logger.info(`OpenAPIValidator Init no of files to cache: ${fileNames?.length}`); + noOfFileToCache = fileNames.length; + } else { + const files = fs.readdirSync(specFolder); + fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml'))); + noOfFileToCache = Math.min(fileNames.length, 3); //If files to cache is not found in env then we will cache just three file + } + noOfFileToCache = Math.min(noOfFileToCache, cachedFileLimit); + console.log('Cache total files: ', noOfFileToCache); -// Function to initialize and cache the OpenAPI validator middleware -const getOpenApiValidatorMiddleware = (specFile: string) => { - if (!cachedOpenApiValidator || cachedSpecFile !== specFile) { - logger.info( - `Cache Not found for OpenApiValidator middleware. Loading.... ${specFile}` - ); - const apiSpec = loadApiSpec(specFile); - cachedOpenApiValidator = OpenApiValidator.middleware({ - apiSpec, - validateRequests: true, - validateResponses: false, - $refParser: { - mode: "dereference" + for (let i = 0; i < noOfFileToCache; i++) { + const file = `${specFolder}/${fileNames[i]}`; + if (!OpenApiValidatorMiddleware.cachedOpenApiValidator[file]) { + logger.info(`Intially cache Not found loadApiSpec file. Loading.... ${file}`); + const apiSpec = this.getApiSpec(file); + const requestHandler = OpenApiValidator.middleware({ + apiSpec, + validateRequests: true, + validateResponses: false, + $refParser: { + mode: "dereference" + } + }) + OpenApiValidatorMiddleware.cachedOpenApiValidator[file] = { + apiSpec, + count: 0, + requestHandler + } + await initializeOpenApiValidatorCache(requestHandler); + } } - }); - cachedSpecFile = specFile; + } catch (err) { + logger.error('Error in initializing open API middleware', err); + } } - return cachedOpenApiValidator; + + public getOpenApiMiddleware(specFile: string): express.RequestHandler[] { + try { + let requestHandler: express.RequestHandler[]; + if (OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]) { + const cachedValidator = OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]; + cachedValidator.count = cachedValidator.count > 1000 ? cachedValidator.count : cachedValidator.count + 1; + logger.info(`Cache found for spec ${specFile}`); + requestHandler = cachedValidator.requestHandler; + } else { + const cashedSpec = Object.entries(OpenApiValidatorMiddleware.cachedOpenApiValidator); + const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit; + if (cashedSpec.length >= cachedFileLimit) { + const specWithLeastCount = cashedSpec.reduce((minEntry, currentEntry) => { + return currentEntry[1].count < minEntry[1].count ? currentEntry : minEntry; + }) || cashedSpec[0]; + logger.info(`Cache count reached limit. Deleting from cache.... ${specWithLeastCount[0]}`); + delete OpenApiValidatorMiddleware.cachedOpenApiValidator[specWithLeastCount[0]]; + } + logger.info(`Cache Not found loadApiSpec file. Loading.... ${specFile}`); + const apiSpec = this.getApiSpec(specFile); + OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile] = { + apiSpec, + count: 1, + requestHandler: OpenApiValidator.middleware({ + apiSpec, + validateRequests: true, + validateResponses: false, + $refParser: { + mode: "dereference" + } + }) + } + requestHandler = OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile].requestHandler; + } + const cacheStats = Object.entries(OpenApiValidatorMiddleware.cachedOpenApiValidator).map((cache) => { + return { + count: cache[1].count, + specFile: cache[0] + } + }); + console.table(cacheStats); + return requestHandler; + } catch (err) { + logger.error('Error in getOpenApiMiddleware', err); + return [] + } + }; +} + +const initializeOpenApiValidatorCache = async (stack: any) => { + try { + let actions: string[] = []; + if ( + (getConfig().app.mode === AppMode.bap && + getConfig().app.gateway.mode === GatewayMode.client) || + (getConfig().app.mode === AppMode.bpp && + getConfig().app.gateway.mode === GatewayMode.network) + ) { + actions = Object.keys(RequestActions); + } else { + actions = Object.keys(ResponseActions); + } + + actions.forEach((action) => { + const mockRequest = (body: any) => { + const req = httpMocks.createRequest({ + method: "POST", + url: `/${action}`, + headers: { + "Content-Type": "application/json", + Authorization: uuid_v4() + }, + body: body + }); + + req.app = { + enabled: (setting: any) => { + if ( + setting === "strict routing" || + setting === "case sensitive routing" + ) { + return true; + } + return false; + } + } as any; + return req; + }; + + const reqObj = mockRequest({ + context: { action: `${action}` }, + message: {} + }); + + walkSubstack(stack, reqObj, {}, () => { + return; + }, false); + }); + } catch (error: any) { } }; export const schemaErrorHandler = ( @@ -54,6 +199,7 @@ export const schemaErrorHandler = ( res: Response, next: NextFunction ) => { + logger.error('OpenApiValidator Error', err); if (err instanceof Exception) { next(err); } else { @@ -68,6 +214,28 @@ export const schemaErrorHandler = ( } }; +const walkSubstack = function ( + stack: any, + req: any, + res: any, + next: NextFunction, + reportError = true +) { + if (typeof stack === "function") { + stack = [stack]; + } + const walkStack = function (i: any, err?: any) { + if (err && reportError) { + return schemaErrorHandler(err, req, res, next); + } + if (i >= stack.length) { + return next(); + } + stack[i](req, res, walkStack.bind(null, i + 1)); + }; + walkStack(0); +}; + export const openApiValidatorMiddleware = async ( req: Request, res: Response<{}, Locals>, @@ -76,7 +244,7 @@ export const openApiValidatorMiddleware = async ( const version = req?.body?.context?.core_version ? req?.body?.context?.core_version : req?.body?.context?.version; - let specFile = `schemas/core_${version}.yaml`; + let specFile = `${specFolder}/core_${version}.yaml`; if (getConfig().app.useLayer2Config) { let doesLayer2ConfigExist = false; @@ -86,47 +254,27 @@ export const openApiValidatorMiddleware = async ( try { doesLayer2ConfigExist = ( await fs.promises.readdir( - `${path.join(path.resolve(__dirname, "../../"))}/schemas` + `${path.join(path.resolve(__dirname, "../../"))}/${specFolder}` ) ).includes(layer2ConfigFilename); } catch (error) { doesLayer2ConfigExist = false; } - if (doesLayer2ConfigExist) specFile = `schemas/${layer2ConfigFilename}`; + if (doesLayer2ConfigExist) specFile = `${specFolder}/${layer2ConfigFilename}`; else { if (getConfig().app.mandateLayer2Config) { + const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration` + logger.error(message); return next( new Exception( ExceptionType.Config_AppConfig_Layer2_Missing, - `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`, + message, 422 ) ); } } } - - const openApiValidator = getOpenApiValidatorMiddleware(specFile); - - const walkSubstack = function ( - stack: any, - req: any, - res: any, - next: NextFunction - ) { - if (typeof stack === "function") { - stack = [stack]; - } - const walkStack = function (i: any, err?: any) { - if (err) { - return schemaErrorHandler(err, req, res, next); - } - if (i >= stack.length) { - return next(); - } - stack[i](req, res, walkStack.bind(null, i + 1)); - }; - walkStack(0); - }; + const openApiValidator = OpenApiValidatorMiddleware.getInstance().getOpenApiMiddleware(specFile); walkSubstack([...openApiValidator], req, res, next); }; diff --git a/src/routes/requests.routes.ts b/src/routes/requests.routes.ts index 2063be3..3b76075 100644 --- a/src/routes/requests.routes.ts +++ b/src/routes/requests.routes.ts @@ -1,7 +1,8 @@ import { NextFunction, Request, Response, Router } from "express"; +import fs from "fs"; +import path from "path"; import { RequestActions, - ResponseActions } from "../schemas/configs/actions.app.config.schema"; import { AppMode } from "../schemas/configs/app.config.schema"; import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; @@ -18,10 +19,8 @@ import { bapClientTriggerHandler } from "../controllers/bap.trigger.controller"; import { bppNetworkRequestHandler } from "../controllers/bpp.request.controller"; import { Locals } from "../interfaces/locals.interface"; import { unConfigureActionHandler } from "../controllers/unconfigured.controller"; -import * as OpenApiValidator from "express-openapi-validator"; -import fs from "fs"; -import path from "path"; import { LogLevelEnum } from "../utils/logger.utils"; + export const requestsRouter = Router(); requestsRouter.get("/logs", (req, res) => { @@ -68,7 +67,7 @@ if ( `/${action}`, (req: Request, res: Response<{}, Locals>, next: NextFunction) => { console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` ); next(); }, @@ -84,7 +83,7 @@ if ( (req: any, res: Response<{}, Locals>, next: NextFunction) => { timestampAuthTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` ); next(); }, @@ -96,7 +95,7 @@ if ( async (req: Request, res: Response, next: NextFunction) => { timestampTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` ); next(); }, @@ -140,7 +139,7 @@ if ( `/${action}`, (req: any, res: Response<{}, Locals>, next: NextFunction) => { console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` ); next(); }, @@ -153,11 +152,7 @@ if ( (req: any, res: Response<{}, Locals>, next: NextFunction) => { timestampAuthTracker.end = new Date().valueOf(); console.log( - `############################################ \n ${getConfig().app.mode}-${getConfig().app.gateway.mode - } AUTH Validator started at: ${timestampAuthTracker.start} and ended at: ${timestampAuthTracker.end}. - Total difference is ${timestampAuthTracker.end - timestampAuthTracker.start} milliseconds, - message ID is ${req?.body?.context?.message_id} - action is ${req?.body?.context?.action}\n ############################################` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` ); next(); }, @@ -169,11 +164,7 @@ if ( async (req: Request, res: Response, next: NextFunction) => { timestampTracker.end = new Date().valueOf(); console.log( - `############################################ ${getConfig().app.mode}-${getConfig().app.gateway.mode - } OPENAPI Validator started at: ${timestampTracker.start} and ended at: ${timestampTracker.end}. - Total difference is ${timestampTracker.end - timestampTracker.start} milliseconds, - message ID is ${req?.body?.context?.message_id} - action is ${req?.body?.context?.action}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` ); next(); }, diff --git a/src/routes/responses.routes.ts b/src/routes/responses.routes.ts index 7fd7990..3789dcb 100644 --- a/src/routes/responses.routes.ts +++ b/src/routes/responses.routes.ts @@ -9,10 +9,8 @@ import { import { contextBuilderMiddleware } from "../middlewares/context.middleware"; import { jsonCompressorMiddleware } from "../middlewares/jsonParser.middleware"; import { - schemaErrorHandler, openApiValidatorMiddleware } from "../middlewares/schemaValidator.middleware"; -import * as OpenApiValidator from "express-openapi-validator"; import { ResponseActions } from "../schemas/configs/actions.app.config.schema"; import { AppMode } from "../schemas/configs/app.config.schema"; import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; @@ -41,7 +39,7 @@ if ( `/${action}`, (req: Request, res: Response, next: NextFunction) => { console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` ); next(); }, @@ -54,7 +52,7 @@ if ( (req: any, res: Response, next: NextFunction) => { timestampAuthTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` ); next(); }, @@ -66,11 +64,7 @@ if ( async (req: Request, res: Response, next: NextFunction) => { timestampTracker.end = new Date().valueOf(); console.log( - `############################################ ${getConfig().app.mode}-${getConfig().app.gateway.mode - } OPENAPI Validator started at: ${timestampTracker.start} and ended at: ${timestampTracker.end}. - Total difference is ${timestampTracker.end - timestampTracker.start} milliseconds, - message ID is ${req?.body?.context?.message_id} - action is ${req?.body?.context?.action}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` ); next(); }, @@ -115,7 +109,7 @@ if ( `/${action}`, (req: any, res: Response, next: NextFunction) => { console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` ); next(); }, @@ -131,7 +125,7 @@ if ( (req: Request, res: Response, next: NextFunction) => { timestampAuthTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` ); next(); }, @@ -143,7 +137,7 @@ if ( async (req: Request, res: Response, next: NextFunction) => { timestampTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` ); next(); }, diff --git a/src/schemas/configs/app.config.schema.ts b/src/schemas/configs/app.config.schema.ts index 180018e..25f656d 100644 --- a/src/schemas/configs/app.config.schema.ts +++ b/src/schemas/configs/app.config.schema.ts @@ -57,6 +57,10 @@ export const appConfigSchema = z.object({ }).optional(), useHMACForWebhook: z.boolean().optional(), sharedKeyForWebhookHMAC: z.string().optional(), + openAPIValidator: z.object({ + cachedFileLimit: z.number().optional(), + initialFilesToCache: z.string().optional() + }).optional() }); export type AppConfigDataType = z.infer; diff --git a/src/utils/becknRequester.utils.ts b/src/utils/becknRequester.utils.ts index c360739..8a00d56 100644 --- a/src/utils/becknRequester.utils.ts +++ b/src/utils/becknRequester.utils.ts @@ -78,11 +78,11 @@ export async function callNetwork( ); if (getConfig().app.mode.toLowerCase() === 'bap') { console.log( - `TMTR - ${body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` + `TMTR - ${body?.context?.message_id} - ${body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` ); } else { console.log( - `TMTR - ${body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` + `TMTR - ${body?.context?.message_id} - ${body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` ); } diff --git a/src/utils/callback.utils.ts b/src/utils/callback.utils.ts index 3e4f664..3a34e99 100644 --- a/src/utils/callback.utils.ts +++ b/src/utils/callback.utils.ts @@ -26,7 +26,7 @@ async function makeClientCallback(data: any, axios_config?: AxiosRequestConfig) .connection as WebhookClientConfigDataType; logger.info(`\nWebhook Triggered on:==> ${clientConnectionConfig.url}\n\n`); console.log( - `TMTR - ${data?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` + `TMTR - ${data?.context?.message_id} - ${data?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` ); const response = await axios.post(clientConnectionConfig.url, data, axios_config || {}); logger.info( diff --git a/src/utils/syncResponses.utils.ts b/src/utils/syncResponses.utils.ts index 565afb2..14ffbcb 100644 --- a/src/utils/syncResponses.utils.ts +++ b/src/utils/syncResponses.utils.ts @@ -60,7 +60,7 @@ export async function sendSyncResponses( return; } console.log( - `TMTR - ${context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` + `TMTR - ${context?.message_id} - ${context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` ); res.status(200).json({ context,