From 33ce1c48be18931961325f216da3116b27b167cd Mon Sep 17 00:00:00 2001 From: Abhishek Y Date: Fri, 14 Jun 2024 14:40:17 +0530 Subject: [PATCH 1/2] Implemented logic to load OpenAPI validator spec at app load to improve latency issue 1. Implemented logic to load OpenAPI validator spec at app load to improve latency issue 2. Removed unused import from some files 3. Added /status routs to debug version of app deployed 4. Added log for OpenAPI validator failure 5. Added log for layer2 config message 6. Added action in latency logs --- config/config-sample-client-localhost.yaml | 33 ++- config/config-sample-network-localhost.yaml | 34 ++- config/config-sample.yaml | 28 +- config/new-config-sample.yaml | 28 +- config/samples/bap-client.yaml | 46 ++-- config/samples/bap-network.yaml | 46 ++-- config/samples/bpp-client.yaml | 50 ++-- config/samples/bpp-network.yaml | 50 ++-- package.json | 4 +- src/app.ts | 19 +- src/controllers/bap.response.controller.ts | 21 +- src/controllers/bap.trigger.controller.ts | 7 +- src/controllers/bpp.request.controller.ts | 10 +- src/controllers/bpp.response.controller.ts | 9 +- src/middlewares/schemaValidator.middleware.ts | 257 ++++++++++++++---- src/routes/requests.routes.ts | 27 +- src/routes/responses.routes.ts | 18 +- src/schemas/configs/app.config.schema.ts | 3 + src/utils/becknRequester.utils.ts | 4 +- src/utils/callback.utils.ts | 2 +- src/utils/syncResponses.utils.ts | 2 +- 21 files changed, 441 insertions(+), 257 deletions(-) diff --git a/config/config-sample-client-localhost.yaml b/config/config-sample-client-localhost.yaml index 8670a9f..4a924df 100644 --- a/config/config-sample-client-localhost.yaml +++ b/config/config-sample-client-localhost.yaml @@ -21,13 +21,13 @@ responseCache: # 2. webhook # 3. pubSub client: - synchronous: + synchronous: mongoURL: "mongodb://127.0.0.1:27017/protocolserver-v2-response-cache" #webhook: # url: "https://beckn.free.beeceptor.com/clientURL" - - #messageQueue: + + #messageQueue: # amqpURL: "amqp://guest:guest@localhost:5672" # incomingQueue: "protocol-server-incoming" # outgoingQueue: "protocol-server-outgoing" @@ -49,25 +49,24 @@ app: actions: requests: search: - ttl : "PT10S" + ttl: "PT10S" init: - ttl : "PT10S" + ttl: "PT10S" select: - ttl : "PT10S" + ttl: "PT10S" confirm: - ttl : "PT10S" + ttl: "PT10S" status: ttl: "PT10S" get_cancellation_reasons: ttl: "PT10S" get_rating_categories: ttl: "PT10S" - - - responses: + + responses: on_search: ttl: "PT10S" - + on_init: ttl: "PT10S" on_select: @@ -80,7 +79,7 @@ app: ttl: "PT10S" rating_categories: ttl: "PT10S" - + # Mandatory. privateKey: "private key" publicKey: "public key" @@ -88,12 +87,12 @@ app: # Mandatory. subscriberId: "dev.bap.faiz.protocol-server.com.dsep:jobs.BAP" subscriberUri: "https://410e-45-248-79-20.ngrok.io/" - + # Mandatory. registryUrl: https://registry.becknprotocol.io/subscribers auth: false uniqueKey: "dev.bap.faiz.protocol-server.key" - + # Mandatory. city: "std:080" country: "IND" @@ -111,3 +110,9 @@ app: # In minutes syncInterval: 30 redis_db: 3 + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 diff --git a/config/config-sample-network-localhost.yaml b/config/config-sample-network-localhost.yaml index 7028c6a..39e75f9 100644 --- a/config/config-sample-network-localhost.yaml +++ b/config/config-sample-network-localhost.yaml @@ -21,13 +21,13 @@ responseCache: # 2. webhook # 3. pubSub client: - synchronous: + synchronous: mongoURL: "mongodb://127.0.0.1:27017/protocolserver-v2-response-cache" #webhook: # url: "https://beckn.free.beeceptor.com/clientURL" - - #messageQueue: + + #messageQueue: # amqpURL: "amqp://guest:guest@localhost:5672" # incomingQueue: "protocol-server-incoming" # outgoingQueue: "protocol-server-outgoing" @@ -49,24 +49,24 @@ app: actions: requests: search: - ttl : "PT10S" + ttl: "PT10S" init: - ttl : "PT10S" + ttl: "PT10S" select: - ttl : "PT10S" + ttl: "PT10S" confirm: - ttl : "PT10S" + ttl: "PT10S" status: - ttl : "PT10S" + ttl: "PT10S" get_cancellation_reasons: ttl: "PT10S" get_rating_categories: ttl: "PT10S" - - responses: + + responses: on_search: ttl: "PT10S" - + on_init: ttl: "PT10S" on_select: @@ -79,7 +79,7 @@ app: ttl: "PT10S" rating_categories: ttl: "PT10S" - + # Mandatory. privateKey: "private key" publicKey: "public key" @@ -87,12 +87,12 @@ app: # Mandatory. subscriberId: "dev.bap.faiz.protocol-server.com.dsep:jobs.BAP" subscriberUri: "https://410e-45-248-79-20.ngrok.io/" - + # Mandatory. registryUrl: https://registry.becknprotocol.io/subscribers auth: true uniqueKey: "dev.bap.faiz.protocol-server.key" - + # Mandatory. city: "std:080" country: "IND" @@ -110,3 +110,9 @@ app: # In minutes syncInterval: 30 redis_db: 3 + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 diff --git a/config/config-sample.yaml b/config/config-sample.yaml index 24fe328..3dddfbd 100644 --- a/config/config-sample.yaml +++ b/config/config-sample.yaml @@ -21,13 +21,13 @@ responseCache: # 2. webhook # 3. pubSub client: - synchronous: + synchronous: mongoURL: "mongodb://tvast:password@localhost:27017/protocol_server-v2?authSource=admin" webhook: url: "https://beckn.free.beeceptor.com/clientURL" - - messageQueue: + + messageQueue: amqpURL: "amqp://guest:guest@localhost:5672" incomingQueue: "protocol-server-incoming" outgoingQueue: "protocol-server-outgoing" @@ -49,17 +49,17 @@ app: actions: requests: search: - ttl : "PT10S" + ttl: "PT10S" init: - ttl : "PT10S" - - responses: + ttl: "PT10S" + + responses: on_search: ttl: "PT10S" - + on_init: ttl: "PT10S" - + # Mandatory. privateKey: "your private key" publicKey: "your public key" @@ -67,12 +67,12 @@ app: # Mandatory. subscriberId: "dev.bap.protocol-server.com" subscriberUri: "https://ayush.free.beeceptor.com/" - + # Mandatory. registryUrl: https://registry.becknprotocol.io/subscribers auth: true uniqueKey: "dev.bap.protocol-server.key" - + # Mandatory. city: "std:080" country: "IND" @@ -90,3 +90,9 @@ app: # In minutes syncInterval: 30 redis_db: 3 + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 diff --git a/config/new-config-sample.yaml b/config/new-config-sample.yaml index bcd08d5..21d72bf 100644 --- a/config/new-config-sample.yaml +++ b/config/new-config-sample.yaml @@ -21,13 +21,13 @@ responseCache: # 2. webhook # 3. pubSub client: - synchronous: + synchronous: mongoURL: "mongodb://tvast:password@localhost:27017/protocol_server-v2?authSource=admin" webhook: url: "https://beckn.free.beeceptor.com/clientURL" - - messageQueue: + + messageQueue: amqpUrl: "amqp://guest:guest@localhost:5672" request-queue: "bap-protocol-server" response-queue: "bap-protocol-client" @@ -50,17 +50,17 @@ app: actions: requests: search: - ttl : "PT10S" + ttl: "PT10S" init: - ttl : "PT10S" - - responses: + ttl: "PT10S" + + responses: on_search: ttl: "PT10S" - + on_init: ttl: "PT10S" - + # Mandatory. privateKey: "your private key" publicKey: "your public key" @@ -68,12 +68,12 @@ app: # Mandatory. subscriberId: "dev.bap.protocol-server.com" subscriberUri: "https://ayush.free.beeceptor.com/" - + # Mandatory. registryUrl: https://registry.becknprotocol.io/subscribers auth: true uniqueKey: "dev.bap.protocol-server.key" - + # Mandatory. city: "std:080" country: "IND" @@ -91,3 +91,9 @@ app: # In minutes syncInterval: 30 redis_db: 3 + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 diff --git a/config/samples/bap-client.yaml b/config/samples/bap-client.yaml index 076ec3d..2cf730e 100644 --- a/config/samples/bap-client.yaml +++ b/config/samples/bap-client.yaml @@ -21,13 +21,13 @@ responseCache: # 2. webhook # 3. pubSub client: - synchronous: + synchronous: mongoURL: "mongodb://MONGO_USERNAME:MONGO_PASSWORD@MONOG_URL:27017/MONGO_DB_NAME?authSource=admin" #webhook: # url: "https://beckn.free.beeceptor.com/clientURL" - - #messageQueue: + + #messageQueue: # amqpURL: "amqp://guest:guest@localhost:5672" # incomingQueue: "protocol-server-incoming" # outgoingQueue: "protocol-server-outgoing" @@ -49,33 +49,33 @@ app: actions: requests: search: - ttl : "PT15S" + ttl: "PT15S" init: - ttl : "PT10S" + ttl: "PT10S" select: - ttl : "PT10S" + ttl: "PT10S" confirm: - ttl : "PT10S" + ttl: "PT10S" status: - ttl : "PT10S" + ttl: "PT10S" track: - ttl : "PT10S" + ttl: "PT10S" cancel: - ttl : "PT10S" + ttl: "PT10S" update: - ttl : "PT10S" + ttl: "PT10S" rating: - ttl : "PT10S" + ttl: "PT10S" support: - ttl : "PT10S" + ttl: "PT10S" get_cancellation_reasons: - ttl : "PT10S" + ttl: "PT10S" get_rating_categories: - ttl : "PT10S" + ttl: "PT10S" cancellation: - ttl : "PT10S" + ttl: "PT10S" - responses: + responses: on_search: ttl: "PT15S" on_init: @@ -99,8 +99,8 @@ app: cancellation_reasons: ttl: "PT10S" rating_categories: - ttl: "PT10S" - + ttl: "PT10S" + # Mandatory. privateKey: "PRIVATE_KEY" publicKey: "PUBLIC_KEY" @@ -113,7 +113,7 @@ app: registryUrl: REGISTRY_URL auth: false uniqueKey: "BAP_SUBSCRIBER_ID_KEY" - + # Mandatory. city: "std:080" country: "IND" @@ -131,3 +131,9 @@ app: # In minutes syncInterval: 30 redis_db: 3 + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 diff --git a/config/samples/bap-network.yaml b/config/samples/bap-network.yaml index a935a71..635c807 100644 --- a/config/samples/bap-network.yaml +++ b/config/samples/bap-network.yaml @@ -21,13 +21,13 @@ responseCache: # 2. webhook # 3. pubSub client: - synchronous: + synchronous: mongoURL: "mongodb://MONGO_USERNAME:MONGO_PASSWORD@MONOG_URL:27017/MONGO_DB_NAME?authSource=admin" #webhook: # url: "https://beckn.free.beeceptor.com/clientURL" - - #messageQueue: + + #messageQueue: # amqpURL: "amqp://guest:guest@localhost:5672" # incomingQueue: "protocol-server-incoming" # outgoingQueue: "protocol-server-outgoing" @@ -49,33 +49,33 @@ app: actions: requests: search: - ttl : "PT15S" + ttl: "PT15S" init: - ttl : "PT10S" + ttl: "PT10S" select: - ttl : "PT10S" + ttl: "PT10S" confirm: - ttl : "PT10S" + ttl: "PT10S" status: - ttl : "PT10S" + ttl: "PT10S" track: - ttl : "PT10S" + ttl: "PT10S" cancel: - ttl : "PT10S" + ttl: "PT10S" update: - ttl : "PT10S" + ttl: "PT10S" rating: - ttl : "PT10S" + ttl: "PT10S" support: - ttl : "PT10S" + ttl: "PT10S" get_cancellation_reasons: - ttl : "PT10S" + ttl: "PT10S" get_rating_categories: - ttl : "PT10S" + ttl: "PT10S" cancellation: - ttl : "PT10S" + ttl: "PT10S" - responses: + responses: on_search: ttl: "PT15S" on_init: @@ -99,8 +99,8 @@ app: cancellation_reasons: ttl: "PT10S" rating_categories: - ttl: "PT10S" - + ttl: "PT10S" + # Mandatory. privateKey: "PRIVATE_KEY" publicKey: "PUBLIC_KEY" @@ -113,7 +113,7 @@ app: registryUrl: REGISTRY_URL auth: false uniqueKey: "BAP_SUBSCRIBER_ID_KEY" - + # Mandatory. city: "std:080" country: "IND" @@ -131,3 +131,9 @@ app: # In minutes syncInterval: 30 redis_db: 3 + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 diff --git a/config/samples/bpp-client.yaml b/config/samples/bpp-client.yaml index 7a90948..b5ec233 100644 --- a/config/samples/bpp-client.yaml +++ b/config/samples/bpp-client.yaml @@ -8,7 +8,7 @@ cache: port: 6379 ttl: "PT10M" # Optional. Default is 0. - db: 0 + db: 0 # Optional. responseCache: @@ -21,13 +21,13 @@ responseCache: # 2. webhook # 3. pubSub client: -# synchronous: -# mongoURL: "mongodb://tvast:password@0.0.0.0:27017/ps?authSource=admin" + # synchronous: + # mongoURL: "mongodb://tvast:password@0.0.0.0:27017/ps?authSource=admin" webhook: - url: "WEBHOOK_URL" - - #messageQueue: + url: "WEBHOOK_URL" + + #messageQueue: # amqpURL: "amqp://guest:guest@localhost:5672" # incomingQueue: "protocol-server-incoming" # outgoingQueue: "protocol-server-outgoing" @@ -49,31 +49,31 @@ app: actions: requests: search: - ttl : "PT15S" + ttl: "PT15S" init: - ttl : "PT10S" + ttl: "PT10S" select: - ttl : "PT10S" + ttl: "PT10S" confirm: - ttl : "PT10S" + ttl: "PT10S" status: - ttl : "PT10S" + ttl: "PT10S" track: - ttl : "PT10S" + ttl: "PT10S" cancel: - ttl : "PT10S" + ttl: "PT10S" update: - ttl : "PT10S" + ttl: "PT10S" rating: - ttl : "PT10S" + ttl: "PT10S" support: - ttl : "PT10S" + ttl: "PT10S" get_cancellation_reasons: ttl: "PT10S" get_rating_categories: - ttl: "PT10S" - - responses: + ttl: "PT10S" + + responses: on_search: ttl: "PT15S" on_init: @@ -97,8 +97,8 @@ app: cancellation_reasons: ttl: "PT10S" rating_categories: - ttl: "PT10S" - + ttl: "PT10S" + # Mandatory. privateKey: "PRIVATE_KEY" publicKey: "PUBLIC_KEY" @@ -111,7 +111,7 @@ app: registryUrl: REGISTRY_URL auth: true uniqueKey: "BPP_SUBSCRIBER_ID_KEY" - + # Mandatory. city: "std:080" country: "IND" @@ -129,3 +129,9 @@ app: # In minutes syncInterval: 30 redis_db: 3 + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 diff --git a/config/samples/bpp-network.yaml b/config/samples/bpp-network.yaml index 857a315..e71e187 100644 --- a/config/samples/bpp-network.yaml +++ b/config/samples/bpp-network.yaml @@ -8,7 +8,7 @@ cache: port: 6379 ttl: "PT10M" # Optional. Default is 0. - db: 0 + db: 0 # Optional. responseCache: @@ -21,13 +21,13 @@ responseCache: # 2. webhook # 3. pubSub client: -# synchronous: -# mongoURL: "mongodb://tvast:password@0.0.0.0:27017/ps?authSource=admin" + # synchronous: + # mongoURL: "mongodb://tvast:password@0.0.0.0:27017/ps?authSource=admin" webhook: - url: "WEBHOOK_URL" - - #messageQueue: + url: "WEBHOOK_URL" + + #messageQueue: # amqpURL: "amqp://guest:guest@localhost:5672" # incomingQueue: "protocol-server-incoming" # outgoingQueue: "protocol-server-outgoing" @@ -49,31 +49,31 @@ app: actions: requests: search: - ttl : "PT15S" + ttl: "PT15S" init: - ttl : "PT10S" + ttl: "PT10S" select: - ttl : "PT10S" + ttl: "PT10S" confirm: - ttl : "PT10S" + ttl: "PT10S" status: - ttl : "PT10S" + ttl: "PT10S" track: - ttl : "PT10S" + ttl: "PT10S" cancel: - ttl : "PT10S" + ttl: "PT10S" update: - ttl : "PT10S" + ttl: "PT10S" rating: - ttl : "PT10S" + ttl: "PT10S" support: - ttl : "PT10S" + ttl: "PT10S" get_cancellation_reasons: ttl: "PT10S" get_rating_categories: - ttl: "PT10S" - - responses: + ttl: "PT10S" + + responses: on_search: ttl: "PT15S" on_init: @@ -97,8 +97,8 @@ app: cancellation_reasons: ttl: "PT10S" rating_categories: - ttl: "PT10S" - + ttl: "PT10S" + # Mandatory. privateKey: "PRIVATE_KEY" publicKey: "PUBLIC_KEY" @@ -111,7 +111,7 @@ app: registryUrl: REGISTRY_URL auth: true uniqueKey: "BPP_SUBSCRIBER_ID_KEY" - + # Mandatory. city: "std:080" country: "IND" @@ -129,3 +129,9 @@ app: # In minutes syncInterval: 30 redis_db: 3 + + useLayer2Config: false + mandateLayer2Config: false + + openAPIValidator: + cachedFileLimit: 5 diff --git a/package.json b/package.json index db22d37..06801ba 100644 --- a/package.json +++ b/package.json @@ -33,14 +33,16 @@ "dotenv": "^16.4.1", "express": "^4.18.1", "express-openapi-validator": "^5.1.6", - "yaml": "^2.4.2", + "express-status-monitor": "^1.3.4", "ioredis": "^5.0.6", "libsodium-wrappers": "^0.7.9", "mongodb": "^4.7.0", + "node-mocks-http": "^1.14.1", "request-ip": "^3.3.0", "uuid": "^8.3.2", "winston": "^3.7.2", "winston-daily-rotate-file": "^4.7.1", + "yaml": "^2.4.2", "zod": "^3.14.2" } } diff --git a/src/app.ts b/src/app.ts index d5b2752..bd25909 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,19 +3,16 @@ import cors from "cors"; import { Exception } from "./models/exception.model"; import { BecknErrorDataType, - becknErrorSchema, BecknErrorType, } from "./schemas/becknError.schema"; -import { RequestActions } from "./schemas/configs/actions.app.config.schema"; import { LookupCache } from "./utils/cache/lookup.cache.utils"; import { RequestCache } from "./utils/cache/request.cache.utils"; import { ResponseCache } from "./utils/cache/response.cache.utils"; -import { SyncCache } from "./utils/cache/sync.cache.utils"; import { ClientUtils } from "./utils/client.utils"; - import { getConfig } from "./utils/config.utils"; import { GatewayUtils } from "./utils/gateway.utils"; import logger from "./utils/logger.utils"; +import { OpenApiValidatorMiddleware } from "./middlewares/schemaValidator.middleware"; const app = Express(); @@ -28,6 +25,15 @@ app.use( const initializeExpress = async (successCallback: Function) => { const app = Express(); + app.use( + require("express-status-monitor")({ + path: "/process" + }) + ); + app.get("/status", async (req: Request, res: Response, next: NextFunction) => { + res.status(200).send('Added logic to cache OpenAPI validator spec on app load'); + }); + // Enabling Cors app.options( "*", @@ -127,10 +133,11 @@ const main = async () => { logger.info("Mode: " + getConfig().app.mode.toLocaleUpperCase()); logger.info( "Gateway Type: " + - getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) + - getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) + getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) + + getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) ); }); + await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware(); } catch (err) { if (err instanceof Exception) { logger.error(err.toString()); diff --git a/src/controllers/bap.response.controller.ts b/src/controllers/bap.response.controller.ts index 5bb8a0a..ecfda5c 100644 --- a/src/controllers/bap.response.controller.ts +++ b/src/controllers/bap.response.controller.ts @@ -1,8 +1,6 @@ import { NextFunction, Request, Response } from "express"; -import { Locals } from "../interfaces/locals.interface"; -import { ResponseActions } from "../schemas/configs/actions.app.config.schema"; -import logger from "../utils/logger.utils"; import * as AmqbLib from "amqplib"; +import moment from "moment"; import { Exception, ExceptionType } from "../models/exception.model"; import { ActionUtils } from "../utils/actions.utils"; import { RequestCache } from "../utils/cache/request.cache.utils"; @@ -21,7 +19,9 @@ import { createTelemetryEvent, processTelemetry, } from "../utils/telemetry.utils"; -import moment from "moment"; +import { Locals } from "../interfaces/locals.interface"; +import { ResponseActions } from "../schemas/configs/actions.app.config.schema"; +import logger from "../utils/logger.utils"; export const bapNetworkResponseHandler = async ( req: Request, @@ -67,7 +67,8 @@ export const bapNetworkResponseHandler = async ( await GatewayUtils.getInstance().sendToClientSideGateway(req.body); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode + } REV EXIT: ${new Date().valueOf()}` ); } catch (err) { let exception: Exception | null = null; @@ -94,8 +95,7 @@ export const bapNetworkResponseSettler = async ( logger.info( "Protocol Client Server (Network Settler) recieving message from inbox queue" ); - - const responseBody = JSON.parse(message?.content.toString()!); + let responseBody = JSON.parse(message?.content.toString()!); logger.info( `Response from BPP NETWORK:\n ${JSON.stringify(responseBody)}\n\n` @@ -106,7 +106,8 @@ export const bapNetworkResponseSettler = async ( responseBody.context.action ); console.log( - `TMTR - ${message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` + `TMTR - ${message_id} - ${action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode + } REV ENTRY: ${new Date().valueOf()}` ); const unsolicitedWebhookUrl = getConfig().app.unsolicitedWebhook?.url; const requestCache = await RequestCache.getInstance().check( @@ -114,6 +115,10 @@ export const bapNetworkResponseSettler = async ( action ); if (!requestCache && unsolicitedWebhookUrl) { + responseBody = { + context: responseBody.context, + responses: [responseBody] + }; unsolicitedCallback(responseBody); return; } diff --git a/src/controllers/bap.trigger.controller.ts b/src/controllers/bap.trigger.controller.ts index 14cda63..b8a5586 100644 --- a/src/controllers/bap.trigger.controller.ts +++ b/src/controllers/bap.trigger.controller.ts @@ -1,8 +1,8 @@ import { NextFunction, Request, Response } from "express"; +import * as AmqbLib from "amqplib"; import { Locals } from "../interfaces/locals.interface"; import { RequestActions } from "../schemas/configs/actions.app.config.schema"; import logger from "../utils/logger.utils"; -import * as AmqbLib from "amqplib"; import { acknowledgeACK, acknowledgeNACK @@ -27,6 +27,7 @@ import { createTelemetryEvent, processTelemetry } from "../utils/telemetry.utils"; + const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`; export const bapClientTriggerHandler = async ( @@ -73,7 +74,7 @@ export const bapClientTriggerHandler = async ( logger.info(`Request from client:\n ${JSON.stringify(req.body)}\n`); await GatewayUtils.getInstance().sendToNetworkSideGateway(req.body); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` ); if (getConfig().client.type == ClientConfigType.synchronous) { sendSyncResponses( @@ -107,7 +108,7 @@ export const bapClientTriggerSettler = async ( try { const body = (JSON.parse(message?.content.toString()!) as any) console.log( - `TMTR - ${body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` + `TMTR - ${body?.context?.message_id} - ${body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` ); logger.info( "Protocol Network Server (Client Settler) recieving message from outbox queue" diff --git a/src/controllers/bpp.request.controller.ts b/src/controllers/bpp.request.controller.ts index cb094ab..ed0166d 100644 --- a/src/controllers/bpp.request.controller.ts +++ b/src/controllers/bpp.request.controller.ts @@ -1,16 +1,14 @@ import { NextFunction, Request, Response } from "express"; +import * as AmqbLib from "amqplib"; +import moment from "moment"; import { RequestActions } from "../schemas/configs/actions.app.config.schema"; import logger from "../utils/logger.utils"; -import * as AmqbLib from "amqplib"; import { Exception, ExceptionType } from "../models/exception.model"; import { acknowledgeACK } from "../utils/acknowledgement.utils"; import { GatewayUtils } from "../utils/gateway.utils"; -import { ActionUtils } from "../utils/actions.utils"; import { RequestCache } from "../utils/cache/request.cache.utils"; import { parseRequestCache } from "../schemas/cache/request.cache.schema"; -import { getSubscriberDetails } from "../utils/lookup.utils"; import { Locals } from "../interfaces/locals.interface"; -import moment from "moment"; import { getConfig } from "../utils/config.utils"; import { ClientConfigType } from "../schemas/configs/client.config.schema"; import { requestCallback } from "../utils/callback.utils"; @@ -47,7 +45,7 @@ export const bppNetworkRequestHandler = async ( await processTelemetry(); } console.log( - `TMTR - ${req.body.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` + `TMTR - ${req.body.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` ); await GatewayUtils.getInstance().sendToClientSideGateway(req.body); } catch (err) { @@ -73,7 +71,7 @@ export const bppNetworkRequestSettler = async ( try { const requestBody = JSON.parse(msg?.content.toString()!); console.log( - `TMTR - ${requestBody?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` + `TMTR - ${requestBody?.context?.message_id} - ${requestBody?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` ); // Generate Telemetry if enabled if (getConfig().app.telemetry.enabled && getConfig().app.telemetry.url) { diff --git a/src/controllers/bpp.response.controller.ts b/src/controllers/bpp.response.controller.ts index 473a550..377901c 100644 --- a/src/controllers/bpp.response.controller.ts +++ b/src/controllers/bpp.response.controller.ts @@ -1,11 +1,11 @@ import { Request, Response, NextFunction } from "express"; +import * as AmqbLib from "amqplib"; +import moment from "moment"; import { Locals } from "../interfaces/locals.interface"; import { - RequestActions, ResponseActions } from "../schemas/configs/actions.app.config.schema"; import logger from "../utils/logger.utils"; -import * as AmqbLib from "amqplib"; import { Exception, ExceptionType } from "../models/exception.model"; import { GatewayUtils } from "../utils/gateway.utils"; import { RequestCache } from "../utils/cache/request.cache.utils"; @@ -28,7 +28,6 @@ import { processTelemetry } from "../utils/telemetry.utils"; import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; -import moment from "moment"; import { getSubscriberDetails } from "../utils/lookup.utils"; export const bppClientResponseHandler = async ( @@ -41,7 +40,7 @@ export const bppClientResponseHandler = async ( acknowledgeACK(res, req.body.context); await GatewayUtils.getInstance().sendToNetworkSideGateway(req.body); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` ); } catch (err) { let exception: Exception | null = null; @@ -68,7 +67,7 @@ export const bppClientResponseSettler = async ( const context = JSON.parse(JSON.stringify(responseBody.context)); const message_id = responseBody.context.message_id; console.log( - `TMTR - ${context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` + `TMTR - ${context?.message_id} - ${context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` ); const requestAction = ActionUtils.getCorrespondingRequestAction( responseBody.context.action diff --git a/src/middlewares/schemaValidator.middleware.ts b/src/middlewares/schemaValidator.middleware.ts index e0fdab8..a5ebb13 100644 --- a/src/middlewares/schemaValidator.middleware.ts +++ b/src/middlewares/schemaValidator.middleware.ts @@ -1,51 +1,184 @@ -import { NextFunction, Request, Response } from "express"; +import express, { NextFunction, Request, Response } from "express"; import * as OpenApiValidator from "express-openapi-validator"; -import { Exception, ExceptionType } from "../models/exception.model"; -import { Locals } from "../interfaces/locals.interface"; -import { getConfig } from "../utils/config.utils"; import fs from "fs"; import path from "path"; import { OpenAPIV3 } from "express-openapi-validator/dist/framework/types"; import YAML from "yaml"; -const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`; -import express from "express"; +import * as httpMocks from "node-mocks-http"; +import { v4 as uuid_v4 } from "uuid"; +import { Exception, ExceptionType } from "../models/exception.model"; +import { Locals } from "../interfaces/locals.interface"; +import { getConfig } from "../utils/config.utils"; import logger from "../utils/logger.utils"; +import { AppMode } from "../schemas/configs/app.config.schema"; +import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; +import { + RequestActions, + ResponseActions +} from "../schemas/configs/actions.app.config.schema"; + +const protocolServerLevel = `${getConfig().app.mode.toUpperCase()}-${getConfig().app.gateway.mode.toUpperCase()}`; +const specFolder = 'schemas'; + +export class OpenApiValidatorMiddleware { + private static instance: OpenApiValidatorMiddleware; + private static cachedOpenApiValidator: { + [filename: string]: { + count: number, + requestHandler: express.RequestHandler[], + apiSpec: OpenAPIV3.Document + } + } = {}; + private static cachedFileLimit: number; -// Cache object -const apiSpecCache: { [filename: string]: OpenAPIV3.Document } = {}; + private constructor() { + OpenApiValidatorMiddleware.cachedFileLimit = getConfig().app.openAPIValidator?.cachedFileLimit || 20; + } + + public static getInstance(): OpenApiValidatorMiddleware { + if (!OpenApiValidatorMiddleware.instance) { + OpenApiValidatorMiddleware.instance = new OpenApiValidatorMiddleware(); + } + return OpenApiValidatorMiddleware.instance; + } -// Function to load and cache the API spec -const loadApiSpec = (specFile: string): OpenAPIV3.Document => { - if (!apiSpecCache[specFile]) { - logger.info(`Cache Not found loadApiSpec file. Loading.... ${specFile}`); + private getApiSpec(specFile: string): OpenAPIV3.Document { const apiSpecYAML = fs.readFileSync(specFile, "utf8"); const apiSpec = YAML.parse(apiSpecYAML); - apiSpecCache[specFile] = apiSpec; - } - return apiSpecCache[specFile]; -}; + return apiSpec; + }; -let cachedOpenApiValidator: express.RequestHandler[] | null = null; -let cachedSpecFile: string | null = null; + public async initOpenApiMiddleware(): Promise { + try { + const files = fs.readdirSync(specFolder); + const fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml'))); + const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit; + logger.info(`OpenAPIValidator Cache count ${cachedFileLimit}`); + for (let i = 0; (i < cachedFileLimit && fileNames[i]); i++) { + const file = `${specFolder}/${fileNames[i]}`; + if (!OpenApiValidatorMiddleware.cachedOpenApiValidator[file]) { + logger.info(`Intially cache Not found loadApiSpec file. Loading.... ${file}`); + const apiSpec = this.getApiSpec(file); + const requestHandler = OpenApiValidator.middleware({ + apiSpec, + validateRequests: true, + validateResponses: false, + $refParser: { + mode: "dereference" + } + }) + OpenApiValidatorMiddleware.cachedOpenApiValidator[file] = { + apiSpec, + count: 0, + requestHandler + } + await initializeOpenApiValidatorCache(requestHandler); + } + } + } catch (err) { + logger.error('Error in initializing open API middleware', err); + } + } -// Function to initialize and cache the OpenAPI validator middleware -const getOpenApiValidatorMiddleware = (specFile: string) => { - if (!cachedOpenApiValidator || cachedSpecFile !== specFile) { - logger.info( - `Cache Not found for OpenApiValidator middleware. Loading.... ${specFile}` - ); - const apiSpec = loadApiSpec(specFile); - cachedOpenApiValidator = OpenApiValidator.middleware({ - apiSpec, - validateRequests: true, - validateResponses: false, - $refParser: { - mode: "dereference" + public getOpenApiMiddleware(specFile: string): express.RequestHandler[] { + try { + let requestHandler: express.RequestHandler[]; + if (OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]) { + const cachedValidator = OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile]; + cachedValidator.count = cachedValidator.count > 1000 ? cachedValidator.count : cachedValidator.count + 1; + logger.info(`Cache found for spec ${specFile}`); + requestHandler = cachedValidator.requestHandler; + } else { + const cashedSpec = Object.entries(OpenApiValidatorMiddleware.cachedOpenApiValidator); + const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit; + if (cashedSpec.length >= cachedFileLimit) { + const specWithLeastCount = cashedSpec.reduce((minEntry, currentEntry) => { + return currentEntry[1].count < minEntry[1].count ? currentEntry : minEntry; + }) || cashedSpec[0]; + logger.info(`Cache count reached limit. Deleting from cache.... ${specWithLeastCount[0]}`); + delete OpenApiValidatorMiddleware.cachedOpenApiValidator[specWithLeastCount[0]]; + } + logger.info(`Cache Not found loadApiSpec file. Loading.... ${specFile}`); + const apiSpec = this.getApiSpec(specFile); + OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile] = { + apiSpec, + count: 1, + requestHandler: OpenApiValidator.middleware({ + apiSpec, + validateRequests: true, + validateResponses: false, + $refParser: { + mode: "dereference" + } + }) + } + requestHandler = OpenApiValidatorMiddleware.cachedOpenApiValidator[specFile].requestHandler; } + const cacheStats = Object.entries(OpenApiValidatorMiddleware.cachedOpenApiValidator).map((cache) => { + return { + count: cache[1].count, + specFile: cache[0] + } + }); + console.table(cacheStats); + return requestHandler; + } catch (err) { + logger.error('Error in getOpenApiMiddleware', err); + return [] + } + }; +} + +const initializeOpenApiValidatorCache = async (stack: any) => { + try { + let actions: string[] = []; + if ( + (getConfig().app.mode === AppMode.bap && + getConfig().app.gateway.mode === GatewayMode.client) || + (getConfig().app.mode === AppMode.bpp && + getConfig().app.gateway.mode === GatewayMode.network) + ) { + actions = Object.keys(RequestActions); + } else { + actions = Object.keys(ResponseActions); + } + + actions.forEach((action) => { + const mockRequest = (body: any) => { + const req = httpMocks.createRequest({ + method: "POST", + url: `/${action}`, + headers: { + "Content-Type": "application/json", + Authorization: uuid_v4() + }, + body: body + }); + + req.app = { + enabled: (setting: any) => { + if ( + setting === "strict routing" || + setting === "case sensitive routing" + ) { + return true; + } + return false; + } + } as any; + return req; + }; + + const reqObj = mockRequest({ + context: { action: `${action}` }, + message: {} + }); + + walkSubstack(stack, reqObj, {}, () => { + return; + }, false); }); - cachedSpecFile = specFile; - } - return cachedOpenApiValidator; + } catch (error: any) { } }; export const schemaErrorHandler = ( @@ -54,6 +187,7 @@ export const schemaErrorHandler = ( res: Response, next: NextFunction ) => { + logger.error('OpenApiValidator Error', err); if (err instanceof Exception) { next(err); } else { @@ -68,6 +202,28 @@ export const schemaErrorHandler = ( } }; +const walkSubstack = function ( + stack: any, + req: any, + res: any, + next: NextFunction, + reportError = true +) { + if (typeof stack === "function") { + stack = [stack]; + } + const walkStack = function (i: any, err?: any) { + if (err && reportError) { + return schemaErrorHandler(err, req, res, next); + } + if (i >= stack.length) { + return next(); + } + stack[i](req, res, walkStack.bind(null, i + 1)); + }; + walkStack(0); +}; + export const openApiValidatorMiddleware = async ( req: Request, res: Response<{}, Locals>, @@ -76,7 +232,7 @@ export const openApiValidatorMiddleware = async ( const version = req?.body?.context?.core_version ? req?.body?.context?.core_version : req?.body?.context?.version; - let specFile = `schemas/core_${version}.yaml`; + let specFile = `${specFolder}/core_${version}.yaml`; if (getConfig().app.useLayer2Config) { let doesLayer2ConfigExist = false; @@ -86,19 +242,21 @@ export const openApiValidatorMiddleware = async ( try { doesLayer2ConfigExist = ( await fs.promises.readdir( - `${path.join(path.resolve(__dirname, "../../"))}/schemas` + `${path.join(path.resolve(__dirname, "../../"))}/${specFolder}` ) ).includes(layer2ConfigFilename); } catch (error) { doesLayer2ConfigExist = false; } - if (doesLayer2ConfigExist) specFile = `schemas/${layer2ConfigFilename}`; + if (doesLayer2ConfigExist) specFile = `${specFolder}/${layer2ConfigFilename}`; else { if (getConfig().app.mandateLayer2Config) { + const message = `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration` + logger.error(message); return next( new Exception( ExceptionType.Config_AppConfig_Layer2_Missing, - `Layer 2 config file ${layer2ConfigFilename} is not installed and it is marked as required in configuration`, + message, 422 ) ); @@ -106,27 +264,6 @@ export const openApiValidatorMiddleware = async ( } } - const openApiValidator = getOpenApiValidatorMiddleware(specFile); - - const walkSubstack = function ( - stack: any, - req: any, - res: any, - next: NextFunction - ) { - if (typeof stack === "function") { - stack = [stack]; - } - const walkStack = function (i: any, err?: any) { - if (err) { - return schemaErrorHandler(err, req, res, next); - } - if (i >= stack.length) { - return next(); - } - stack[i](req, res, walkStack.bind(null, i + 1)); - }; - walkStack(0); - }; + const openApiValidator = OpenApiValidatorMiddleware.getInstance().getOpenApiMiddleware(specFile); walkSubstack([...openApiValidator], req, res, next); }; diff --git a/src/routes/requests.routes.ts b/src/routes/requests.routes.ts index 2063be3..3b76075 100644 --- a/src/routes/requests.routes.ts +++ b/src/routes/requests.routes.ts @@ -1,7 +1,8 @@ import { NextFunction, Request, Response, Router } from "express"; +import fs from "fs"; +import path from "path"; import { RequestActions, - ResponseActions } from "../schemas/configs/actions.app.config.schema"; import { AppMode } from "../schemas/configs/app.config.schema"; import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; @@ -18,10 +19,8 @@ import { bapClientTriggerHandler } from "../controllers/bap.trigger.controller"; import { bppNetworkRequestHandler } from "../controllers/bpp.request.controller"; import { Locals } from "../interfaces/locals.interface"; import { unConfigureActionHandler } from "../controllers/unconfigured.controller"; -import * as OpenApiValidator from "express-openapi-validator"; -import fs from "fs"; -import path from "path"; import { LogLevelEnum } from "../utils/logger.utils"; + export const requestsRouter = Router(); requestsRouter.get("/logs", (req, res) => { @@ -68,7 +67,7 @@ if ( `/${action}`, (req: Request, res: Response<{}, Locals>, next: NextFunction) => { console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` ); next(); }, @@ -84,7 +83,7 @@ if ( (req: any, res: Response<{}, Locals>, next: NextFunction) => { timestampAuthTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` ); next(); }, @@ -96,7 +95,7 @@ if ( async (req: Request, res: Response, next: NextFunction) => { timestampTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` ); next(); }, @@ -140,7 +139,7 @@ if ( `/${action}`, (req: any, res: Response<{}, Locals>, next: NextFunction) => { console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW ENTRY: ${new Date().valueOf()}` ); next(); }, @@ -153,11 +152,7 @@ if ( (req: any, res: Response<{}, Locals>, next: NextFunction) => { timestampAuthTracker.end = new Date().valueOf(); console.log( - `############################################ \n ${getConfig().app.mode}-${getConfig().app.gateway.mode - } AUTH Validator started at: ${timestampAuthTracker.start} and ended at: ${timestampAuthTracker.end}. - Total difference is ${timestampAuthTracker.end - timestampAuthTracker.start} milliseconds, - message ID is ${req?.body?.context?.message_id} - action is ${req?.body?.context?.action}\n ############################################` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` ); next(); }, @@ -169,11 +164,7 @@ if ( async (req: Request, res: Response, next: NextFunction) => { timestampTracker.end = new Date().valueOf(); console.log( - `############################################ ${getConfig().app.mode}-${getConfig().app.gateway.mode - } OPENAPI Validator started at: ${timestampTracker.start} and ended at: ${timestampTracker.end}. - Total difference is ${timestampTracker.end - timestampTracker.start} milliseconds, - message ID is ${req?.body?.context?.message_id} - action is ${req?.body?.context?.action}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` ); next(); }, diff --git a/src/routes/responses.routes.ts b/src/routes/responses.routes.ts index 7fd7990..3789dcb 100644 --- a/src/routes/responses.routes.ts +++ b/src/routes/responses.routes.ts @@ -9,10 +9,8 @@ import { import { contextBuilderMiddleware } from "../middlewares/context.middleware"; import { jsonCompressorMiddleware } from "../middlewares/jsonParser.middleware"; import { - schemaErrorHandler, openApiValidatorMiddleware } from "../middlewares/schemaValidator.middleware"; -import * as OpenApiValidator from "express-openapi-validator"; import { ResponseActions } from "../schemas/configs/actions.app.config.schema"; import { AppMode } from "../schemas/configs/app.config.schema"; import { GatewayMode } from "../schemas/configs/gateway.app.config.schema"; @@ -41,7 +39,7 @@ if ( `/${action}`, (req: Request, res: Response, next: NextFunction) => { console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` ); next(); }, @@ -54,7 +52,7 @@ if ( (req: any, res: Response, next: NextFunction) => { timestampAuthTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` ); next(); }, @@ -66,11 +64,7 @@ if ( async (req: Request, res: Response, next: NextFunction) => { timestampTracker.end = new Date().valueOf(); console.log( - `############################################ ${getConfig().app.mode}-${getConfig().app.gateway.mode - } OPENAPI Validator started at: ${timestampTracker.start} and ended at: ${timestampTracker.end}. - Total difference is ${timestampTracker.end - timestampTracker.start} milliseconds, - message ID is ${req?.body?.context?.message_id} - action is ${req?.body?.context?.action}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` ); next(); }, @@ -115,7 +109,7 @@ if ( `/${action}`, (req: any, res: Response, next: NextFunction) => { console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV ENTRY: ${new Date().valueOf()}` ); next(); }, @@ -131,7 +125,7 @@ if ( (req: Request, res: Response, next: NextFunction) => { timestampAuthTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} AUTH: ${timestampAuthTracker.end - timestampAuthTracker.start} ms` ); next(); }, @@ -143,7 +137,7 @@ if ( async (req: Request, res: Response, next: NextFunction) => { timestampTracker.end = new Date().valueOf(); console.log( - `TMTR - ${req?.body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` + `TMTR - ${req?.body?.context?.message_id} - ${req?.body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} OPENAPI Val: ${timestampTracker.end - timestampTracker.start} ms` ); next(); }, diff --git a/src/schemas/configs/app.config.schema.ts b/src/schemas/configs/app.config.schema.ts index 4160bae..1d91995 100644 --- a/src/schemas/configs/app.config.schema.ts +++ b/src/schemas/configs/app.config.schema.ts @@ -54,6 +54,9 @@ export const appConfigSchema = z.object({ mandateLayer2Config: z.boolean().optional(), unsolicitedWebhook: z.object({ url: z.string().optional() + }).optional(), + openAPIValidator: z.object({ + cachedFileLimit: z.number().optional() }).optional() }); diff --git a/src/utils/becknRequester.utils.ts b/src/utils/becknRequester.utils.ts index c360739..8a00d56 100644 --- a/src/utils/becknRequester.utils.ts +++ b/src/utils/becknRequester.utils.ts @@ -78,11 +78,11 @@ export async function callNetwork( ); if (getConfig().app.mode.toLowerCase() === 'bap') { console.log( - `TMTR - ${body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` + `TMTR - ${body?.context?.message_id} - ${body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` ); } else { console.log( - `TMTR - ${body?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` + `TMTR - ${body?.context?.message_id} - ${body?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` ); } diff --git a/src/utils/callback.utils.ts b/src/utils/callback.utils.ts index ae60010..6fc4625 100644 --- a/src/utils/callback.utils.ts +++ b/src/utils/callback.utils.ts @@ -28,7 +28,7 @@ async function makeClientCallback(data: any) { .connection as WebhookClientConfigDataType; logger.info(`\nWebhook Triggered on:==> ${clientConnectionConfig.url}\n\n`); console.log( - `TMTR - ${data?.context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` + `TMTR - ${data?.context?.message_id} - ${data?.context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} FORW EXIT: ${new Date().valueOf()}` ); const response = await axios.post(clientConnectionConfig.url, data); logger.info( diff --git a/src/utils/syncResponses.utils.ts b/src/utils/syncResponses.utils.ts index 565afb2..14ffbcb 100644 --- a/src/utils/syncResponses.utils.ts +++ b/src/utils/syncResponses.utils.ts @@ -60,7 +60,7 @@ export async function sendSyncResponses( return; } console.log( - `TMTR - ${context?.message_id} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` + `TMTR - ${context?.message_id} - ${context?.action} - ${getConfig().app.mode}-${getConfig().app.gateway.mode} REV EXIT: ${new Date().valueOf()}` ); res.status(200).json({ context, From 5e799f9e179a637f7ae5a556e5cbb555a02c5f70 Mon Sep 17 00:00:00 2001 From: alephseven Date: Fri, 28 Jun 2024 12:27:34 +0530 Subject: [PATCH 2/2] feat: Initial cache load Modified the logic to cache few files upon app load --- src/app.ts | 21 +++++++++---------- src/middlewares/schemaValidator.middleware.ts | 20 ++++++++++++++---- src/schemas/configs/app.config.schema.ts | 3 ++- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/app.ts b/src/app.ts index bd25909..9984297 100644 --- a/src/app.ts +++ b/src/app.ts @@ -22,7 +22,7 @@ app.use( }) ); -const initializeExpress = async (successCallback: Function) => { +const initializeExpress = async () => { const app = Express(); app.use( @@ -114,7 +114,6 @@ const initializeExpress = async (successCallback: Function) => { const PORT: number = getConfig().server.port; app.listen(PORT, () => { logger.info("Protocol Server started on PORT : " + PORT); - successCallback(); }); }; @@ -128,16 +127,16 @@ const main = async () => { await LookupCache.getInstance().initialize(); await RequestCache.getInstance().initialize(); - await initializeExpress(() => { - logger.info("Protocol Server Started Successfully"); - logger.info("Mode: " + getConfig().app.mode.toLocaleUpperCase()); - logger.info( - "Gateway Type: " + - getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) + - getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) - ); - }); + await initializeExpress(); + logger.info("Protocol Server Started Successfully"); + logger.info("Mode: " + getConfig().app.mode.toLocaleUpperCase()); + logger.info( + "Gateway Type: " + + getConfig().app.gateway.mode.toLocaleUpperCase().substring(0, 1) + + getConfig().app.gateway.mode.toLocaleUpperCase().substring(1) + ); await OpenApiValidatorMiddleware.getInstance().initOpenApiMiddleware(); + logger.info('Initialized openapi validator middleware') } catch (err) { if (err instanceof Exception) { logger.error(err.toString()); diff --git a/src/middlewares/schemaValidator.middleware.ts b/src/middlewares/schemaValidator.middleware.ts index a5ebb13..3f9ddee 100644 --- a/src/middlewares/schemaValidator.middleware.ts +++ b/src/middlewares/schemaValidator.middleware.ts @@ -50,11 +50,23 @@ export class OpenApiValidatorMiddleware { public async initOpenApiMiddleware(): Promise { try { - const files = fs.readdirSync(specFolder); - const fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml'))); + let fileToCache = getConfig().app?.openAPIValidator?.initialFilesToCache; + let fileNames, noOfFileToCache = 0; const cachedFileLimit: number = OpenApiValidatorMiddleware.cachedFileLimit; - logger.info(`OpenAPIValidator Cache count ${cachedFileLimit}`); - for (let i = 0; (i < cachedFileLimit && fileNames[i]); i++) { + logger.info(`OpenAPIValidator Total Cache capacity ${cachedFileLimit}`); + if(fileToCache) { + fileNames = fileToCache.split(/\s*,\s*/).map(item => item.trim()); + logger.info(`OpenAPIValidator Init no of files to cache: ${fileNames?.length}`); + noOfFileToCache = fileNames.length; + } else { + const files = fs.readdirSync(specFolder); + fileNames = files.filter(file => fs.lstatSync(path.join(specFolder, file)).isFile() && (file.endsWith('.yaml') || file.endsWith('.yml'))); + noOfFileToCache = Math.min(fileNames.length, 3); //If files to cache is not found in env then we will cache just three file + } + noOfFileToCache = Math.min(noOfFileToCache, cachedFileLimit); + console.log('Cache total files: ', noOfFileToCache); + + for (let i = 0; i < noOfFileToCache; i++) { const file = `${specFolder}/${fileNames[i]}`; if (!OpenApiValidatorMiddleware.cachedOpenApiValidator[file]) { logger.info(`Intially cache Not found loadApiSpec file. Loading.... ${file}`); diff --git a/src/schemas/configs/app.config.schema.ts b/src/schemas/configs/app.config.schema.ts index 1d91995..5b722e1 100644 --- a/src/schemas/configs/app.config.schema.ts +++ b/src/schemas/configs/app.config.schema.ts @@ -56,7 +56,8 @@ export const appConfigSchema = z.object({ url: z.string().optional() }).optional(), openAPIValidator: z.object({ - cachedFileLimit: z.number().optional() + cachedFileLimit: z.number().optional(), + initialFilesToCache: z.string().optional() }).optional() });